#!/usr/bin/env bash
# Vloud public bootstrap — `curl -fsSL https://install.vloud.app | sudo bash`.
#
# Modes (auto-detected; explicit flags override):
#
#   (default, no flag)     — auto.  If /opt/vloud/packages/server/dist
#                            exists, switches to upgrade mode.  Otherwise
#                            runs fresh install.
#   --upgrade              — explicit upgrade. Preserves /etc/vloud.env,
#                            /var/lib/vloud, SQLite DB, generated nginx
#                            vhosts, SSL certs, pm2 deployments. Swaps
#                            binaries atomically via /opt/vloud-staging-<ts>,
#                            health-checks the new engine, rolls back on
#                            failure.
#   --repair               — re-emit systemd units, daemon-reload, restart
#                            services. No state touched. For when the
#                            install dir is fine but systemd got tangled.
#   --doctor / --health    — read-only health check. Prints services,
#                            ports, version, DB integrity. No changes.
#   --force-clean-install  — explicit destructive reinstall.  Stops
#                            services, removes /opt/vloud, /etc/vloud.env,
#                            /var/lib/vloud, /etc/systemd/system/vloud*,
#                            then runs fresh install. Last resort.
#   --uninstall            — remove the Vloud engine + state WITHOUT
#                            reinstalling. Same removals as force-clean
#                            (engine, units, /opt/vloud, /var/lib/vloud,
#                            /etc/vloud*, sudoers, engine nginx vhosts);
#                            keeps the OS, SSH, and daemon-stack packages.
#                            Prompts for a typed "UNINSTALL"; add --yes to
#                            skip the prompt (unattended).
#   --fresh                — explicit fresh install. Refuses to run if
#                            /opt/vloud exists (use --force-clean-install
#                            for the destructive path).
#
# Optional add-ons (combine with any install mode):
#
#   --with-dotnet          — install the ASP.NET Core RUNTIME, so a pipeline
#                            can deploy a .NET application with
#                            `runtime: dotnet`. Opt-in on purpose: it is a few
#                            hundred megabytes, most hosts never run a .NET
#                            app, and the engine NEVER installs a runtime
#                            implicitly during a deploy.
#
# Curl-pipe-bash:
#   curl -fsSL https://install.vloud.app | sudo bash
#   curl -fsSL https://install.vloud.app | sudo bash -s -- --upgrade
#   curl -fsSL https://install.vloud.app | sudo bash -s -- --repair
#   curl -fsSL https://install.vloud.app | sudo bash -s -- --doctor
#   curl -fsSL https://install.vloud.app | sudo bash -s -- --force-clean-install
#   curl -fsSL https://install.vloud.app | sudo bash -s -- --uninstall
#   curl -fsSL https://install.vloud.app | sudo bash -s -- --uninstall --yes
#   curl -fsSL https://install.vloud.app | sudo bash -s -- --upgrade --with-dotnet
#
# Fresh-install phases (auto / --fresh / --force-clean-install):
#   1. Pre-flight (OS / arch / disk / RAM / outbound network)
#   2. Apt install (curl, nginx, node 20, sqlite3, dnsutils, certbot, php-fpm)
#   3. Vloud release download + extract to /opt/vloud
#   4. Configure: vloud user, /etc/vloud.env (with trial state), systemd unit
#   5. Start engine + emit dashboard URL + trial status
#   6. Daemon stack (Phase 1) — postfix/dovecot/rspamd/fail2ban/bind/ftp/
#      firewall/tenant-slices/storage-quota via bootstrap-daemon-stack.sh.
#      Default ON; cPanel hosts skip by default; per-daemon opt-out env vars.
#
# Idempotent: every step gates on existing state.  Re-runs are safe.
# Zero-interaction: defaults everywhere, no prompts.
# Logs: /var/log/vloud/install.log
set -euo pipefail

# ─── Defaults (operator can override via env) ───
VLOUD_INSTALL_DIR=${VLOUD_INSTALL_DIR:-/opt/vloud}
VLOUD_RELEASE_URL=${VLOUD_RELEASE_URL:-https://install.vloud.app/releases/latest/vloud-latest.tar.gz}
VLOUD_PORT=${VLOUD_PORT:-2544}
VLOUD_BIND_HOST=${VLOUD_BIND_HOST:-127.0.0.1}
VLOUD_TRIAL_DAYS=${VLOUD_TRIAL_DAYS:-30}
# 0.6.0c — hoisted from the fresh-install env-phase so that
# `do_reconcile_env_file()` (called from --upgrade + --repair via
# the mode dispatcher) can reference it without tripping `set -u`.
# Operator can override via env if pointing at a private license
# server (e.g. for offline / behind-firewall installs).
VLOUD_LICENSE_SERVER_URL_DEFAULT=${VLOUD_LICENSE_SERVER_URL_DEFAULT:-https://license.vloud.app}
ENV_FILE=/etc/vloud.env
LOG_DIR=/var/log/vloud
LOG_FILE=$LOG_DIR/install.log
SYSTEMD_UNIT=/etc/systemd/system/vloud.service
SYSTEMD_SLICE=/etc/systemd/system/vloud.slice
SYSTEMD_WORKER=/etc/systemd/system/vloud-job-worker.service
SYSTEMD_SCHEDULER=/etc/systemd/system/vloud-scheduler.service
# Phase E (2026-05-14): auto-rollback unit fired by vloud.service's
# OnFailure when the engine crash-loops past StartLimitBurst.
SYSTEMD_ROLLBACK=/etc/systemd/system/vloud-rollback.service

# ─── Lifecycle progress + apply-run correlation (PR1) ───
#
# Engine-orchestrated upgrades (POST /api/v1/updates/apply →
# executePlan → spawns bootstrap.sh) pass --apply-run-id=<n> and
# --version=<v>; bootstrap.sh emits append-only JSONL events to
# VLOUD_PROGRESS_FILE so the engine can resolve update_status
# without parsing logs. Operator-run bootstrap.sh (curl-pipe-bash)
# omits these flags and behaves identically to pre-PR1 — every new
# code path is conditional on the flags being set.
VLOUD_PROGRESS_FILE=${VLOUD_PROGRESS_FILE:-/var/lib/vloud/.upgrade-progress}
VLOUD_PROGRESS_RETAIN_BYTES=${VLOUD_PROGRESS_RETAIN_BYTES:-262144}  # 256 KiB before rotation
VLOUD_PROGRESS_STALE_DAYS=${VLOUD_PROGRESS_STALE_DAYS:-30}          # how long rotated logs live
VLOUD_APPLY_RUN_ID=${VLOUD_APPLY_RUN_ID:-}
VLOUD_TARGET_VERSION=${VLOUD_TARGET_VERSION:-}
VLOUD_ROLLBACK_TO=${VLOUD_ROLLBACK_TO:-}
# Phase E (2026-05-14): set to 1 when systemd's vloud-rollback.service
# fires us via OnFailure. Currently informational only.
VLOUD_ROLLBACK_AUTOMATIC=${VLOUD_ROLLBACK_AUTOMATIC:-}
VLOUD_ASSUME_YES=${VLOUD_ASSUME_YES:-0}

# ─── 1. Pre-flight ───
# Root check is mode-aware — --doctor is read-only and runs fine as
# any user that can read systemd state + reach 127.0.0.1:2544. The
# four mutating modes (fresh, upgrade, repair, force-clean) need root.
_NEEDS_ROOT=1
for _a in "$@"; do
  case "$_a" in
    --doctor|--health|--help|-h) _NEEDS_ROOT=0 ;;
  esac
done
if (( _NEEDS_ROOT == 1 )) && [[ $EUID -ne 0 ]]; then
  echo "ERROR: bootstrap.sh must be run as root for this mode (curl -fsSL ... | sudo bash)" >&2
  echo "       (--doctor / --help do not require root)" >&2
  exit 1
fi
unset _NEEDS_ROOT _a

if [[ -f /etc/os-release ]]; then
  . /etc/os-release
else
  echo "ERROR: /etc/os-release missing — unsupported distro" >&2
  exit 1
fi
# OS-family detection. Vloud supports the Debian family (Ubuntu/Debian)
# and the RHEL family (AlmaLinux/Rocky/RHEL 9). OS_FAMILY drives every
# package-manager / firewall / SELinux branch below — see the
# install_packages / ensure_rhel_repos / configure_selinux helpers.
case "$ID" in
  ubuntu|debian)        OS_FAMILY=debian ;;
  almalinux|rocky|rhel) OS_FAMILY=rhel ;;
  *)
    case " ${ID_LIKE:-} " in
      *" rhel "*|*" fedora "*|*" centos "*) OS_FAMILY=rhel ;;
      *" debian "*)                         OS_FAMILY=debian ;;
      *) echo "ERROR: Vloud bootstrap supports Ubuntu/Debian and AlmaLinux/Rocky/RHEL 9 only (you have $ID)" >&2; exit 1 ;;
    esac ;;
esac
export OS_FAMILY
# Major version (e.g. 9 from 9.3) — used for EPEL/Remi repo URLs on RHEL.
OS_VERSION_MAJOR=${VERSION_ID%%.*}
export OS_VERSION_MAJOR
# Web-server group that owns ACME / docroot dirs: www-data on Debian,
# nginx on the RHEL family (matches OsAdapter.paths.httpGroup engine-side).
if [[ "$OS_FAMILY" == rhel ]]; then HTTP_GROUP=nginx; else HTTP_GROUP=www-data; fi
export HTTP_GROUP

# Logging tee — set up before any output so we capture everything.
mkdir -p "$LOG_DIR"
exec > >(tee -a "$LOG_FILE") 2>&1

CYAN='\033[1;36m'; GREEN='\033[1;32m'; YELLOW='\033[1;33m'; RED='\033[1;31m'; RESET='\033[0m'
say()  { printf "${CYAN}▸${RESET} %s\n" "$1"; }
ok()   { printf "  ${GREEN}✓${RESET} %s\n" "$1"; }
warn() { printf "  ${YELLOW}⚠${RESET} %s\n" "$1"; }
fail() { printf "  ${RED}✗${RESET} %s\n" "$1" >&2; }
die()  { fail "$1"; emit_progress die failed "$1"; exit "${2:-1}"; }

# ─── OS-family package / firewall / SELinux abstraction ───
# Every package-manager touch goes through these so the Debian path
# stays byte-identical to the historical apt commands while the RHEL
# path uses dnf. Keep the debian branch EXACTLY as the inline commands
# were — this is a refactor, not a behaviour change for Ubuntu.

pkg_update() {
  case "$OS_FAMILY" in
    debian) DEBIAN_FRONTEND=noninteractive apt-get update -qq ;;
    rhel)   dnf -y -q makecache >/dev/null 2>&1 || true ;;
  esac
}

# Packages this run refused to install because they would have removed
# something, and packages skipped because the host already had them. Reported
# at the end of the install so the operator sees what was left alone.
VLOUD_PKGS_REFUSED=()
VLOUD_PKGS_ALREADY=()

# What would installing these packages REMOVE from this host?
#
# Echoes the package names apt/dnf plans to remove; empty output means the
# transaction is purely additive.
pkg_plan_removals() {
  case "$OS_FAMILY" in
    debian)
      DEBIAN_FRONTEND=noninteractive apt-get install -s -y "$@" 2>/dev/null \
        | awk '/^Remv /{print $2}'
      ;;
    rhel)
      # `--assumeno` prints the transaction table and exits without applying it.
      dnf install --assumeno "$@" 2>/dev/null \
        | awk '/^Removing|^Obsoleting/{f=1;next} /^$|^Transaction Summary/{f=0} f&&NF{print $1}'
      ;;
  esac
}

# Install packages WITHOUT removing anything and WITHOUT disturbing what
# already works.
#
# ## The incident this exists to prevent
#
# This function used to be a bare `apt-get install -y "$@"`. On a clean VPS
# that is fine. On a server that is already doing a job it is not, and on
# 2026-08-27 it did real damage to a live host:
#
#   - `mariadb-server` CONFLICTS with `mysql-server`. apt resolved that
#     conflict exactly as instructed — it REMOVED MySQL 8.0, installed MariaDB
#     10.11, and pointed mysql.service at MariaDB with a brand-new empty
#     datadir. `-y` meant nothing ever asked.
#   - Re-installing nginx, php-fpm, redis and bind that were ALREADY installed
#     and running triggered a mass stop/start. nginx lost the race for its own
#     :443 socket, failed with EADDRINUSE, and — having no Restart= policy —
#     stayed down. The site was refusing connections for 14 minutes.
#   - certbot renewals failed as collateral, because the HTTP-01 challenge
#     needs the web server that was down.
#
# An installer is not entitled to any of that. Two rules now:
#
#   1. NEVER remove a package. If installing X would remove Y, X is dropped and
#      the operator is told. Vloud coming up without an optional component is
#      recoverable; a deleted database server is not.
#   2. NEVER re-install what is already there. A package that is present is
#      left completely alone, so nothing gets reconfigured and nothing gets
#      restarted out from under a running service.
#
# Both rules are deliberately conservative in the same direction: when in
# doubt, change nothing and say so.
install_packages() {
  [[ $# -gt 0 ]] || return 0

  # Rule 2 — drop anything the host already has.
  local wanted=() p
  for p in "$@"; do
    if pkg_installed "$p"; then
      VLOUD_PKGS_ALREADY+=("$p")
    else
      wanted+=("$p")
    fi
  done
  [[ ${#wanted[@]} -gt 0 ]] || return 0

  # Rule 1 — would this transaction remove anything?
  local removals
  removals=$(pkg_plan_removals "${wanted[@]}" | sort -u | tr '\n' ' ')
  if [[ -n "${removals// /}" ]]; then
    # Something in the set is destructive. Find out WHICH, one at a time, so a
    # single bad package does not cost the operator all the others. This path
    # is rare, so the extra simulations do not matter.
    local safe=() bad
    for p in "${wanted[@]}"; do
      bad=$(pkg_plan_removals "$p" | sort -u | tr '\n' ' ')
      if [[ -n "${bad// /}" ]]; then
        VLOUD_PKGS_REFUSED+=("$p")
        warn "refusing to install '$p': it would REMOVE ${bad% }"
        warn "  leaving the existing package(s) in place. Vloud will run without '$p'."
      else
        safe+=("$p")
      fi
    done
    # `${safe[@]+...}` guard: expanding an EMPTY array under `set -u` is an
    # error on bash 3.2, and an installer that aborts here would leave the host
    # half-configured — the opposite of the point.
    wanted=(${safe[@]+"${safe[@]}"})
    # Every requested package was refused: nothing was installed, so report
    # FAILURE. Callers chain alternatives on that — `install_packages
    # aspnetcore-runtime-8.0 || install_packages dotnet-runtime-8.0` must fall
    # through to the second option, not believe the first one succeeded.
    # A partial success still returns 0: the set as a whole made progress, and
    # the top-level call runs under `set -e`, where failing it would abort the
    # whole install over one optional component.
    [[ ${#wanted[@]} -gt 0 ]] || return 1
  fi

  case "$OS_FAMILY" in
    # --no-remove is a second, independent belt: if the simulation and the real
    # run ever disagree (a repo changed underneath us), apt aborts rather than
    # deleting something.
    debian) DEBIAN_FRONTEND=noninteractive apt-get install -y -qq --no-remove "${wanted[@]}" ;;
    rhel)   dnf install -y -q "${wanted[@]}" ;;
  esac
}

# Print what install_packages left alone. Called once, after the package phase.
report_package_decisions() {
  if [[ ${#VLOUD_PKGS_ALREADY[@]} -gt 0 ]]; then
    ok "left ${#VLOUD_PKGS_ALREADY[@]} already-installed package(s) untouched (no restarts): ${VLOUD_PKGS_ALREADY[*]}"
  fi
  if [[ ${#VLOUD_PKGS_REFUSED[@]} -gt 0 ]]; then
    warn "NOT installed, because installing them would have removed existing software:"
    warn "  ${VLOUD_PKGS_REFUSED[*]}"
    warn "  This host keeps what it had. If you want the Vloud-preferred package,"
    warn "  migrate the data yourself first, then install it by hand."
  fi
}

pkg_installed() { # pkg_installed NAME
  case "$OS_FAMILY" in
    # `dpkg -s` alone is not enough: it also succeeds for a package that was
    # REMOVED but left its config behind (status "deinstall ok config-files").
    # install_packages skips whatever this reports as present, so a loose test
    # here would silently skip installing something the host does not actually
    # have.
    debian) dpkg-query -W -f='${db:Status-Status}' "$1" 2>/dev/null | grep -x installed >/dev/null ;;
    rhel)   rpm -q "$1" >/dev/null 2>&1 ;;
  esac
}

# Which PHP does this host actually have? Echoes a bare version
# ("8.3", "8.5") or returns 1 when the repos offer no PHP-FPM at all.
#
# Debian family only — the RHEL branch installs unversioned names from
# the Remi module, where one php-fpm service serves whatever version
# the module selected.
#
# Requires `pkg_update` to have run: apt-cache reads the local package
# lists, so on a never-updated host it would see nothing and this
# would wrongly report no PHP.
# The user the engine actually runs as — read from the unit, never
# assumed. This is the single source of truth for who must own
# /opt/vloud/packages: the engine drops CAP_DAC_OVERRIDE, so being uid 0
# grants it no permission bypass and the database must be owned by
# exactly this user.
#
# Order: the live unit (what systemd will really execute) → the unit
# shipped in the release → root, which is what packaging/systemd
# pins today.
engine_service_user() {
  local u=""
  if command -v systemctl >/dev/null 2>&1; then
    u=$(systemctl show vloud -p User --value 2>/dev/null)
  fi
  if [[ -z "$u" ]]; then
    u=$(sed -n 's/^User=//p' /etc/systemd/system/vloud.service 2>/dev/null | head -1)
  fi
  if [[ -z "$u" ]]; then
    u=$(sed -n 's/^User=//p' "$VLOUD_INSTALL_DIR/packaging/systemd/vloud.service" 2>/dev/null | head -1)
  fi
  [[ -z "$u" ]] && u=root
  # A unit naming a user that does not exist would leave the tree
  # unwritable in a way chown cannot fix; fall back rather than that.
  id -u "$u" >/dev/null 2>&1 || u=root
  printf '%s' "$u"
}

detect_php_version() {
  local preferred=8.3 found
  if apt-cache show "php${preferred}-fpm" >/dev/null 2>&1; then
    echo "$preferred"; return 0
  fi
  # Versioned packages only — the unversioned `php-fpm` metapackage
  # also matches a loose pattern, and it carries no version to build
  # `/etc/php/<v>/fpm/pool.d` paths from.
  found="$(apt-cache search --names-only '^php[0-9]+\.[0-9]+-fpm$' 2>/dev/null \
            | sed -n 's/^php\([0-9][0-9]*\.[0-9][0-9]*\)-fpm .*/\1/p' \
            | sort -V | tail -1)"
  [[ -n "$found" ]] || return 1
  echo "$found"
}

# RHEL needs extra repos for packages Debian ships in base: EPEL
# (certbot, whois, fail2ban, clamav), Remi (PHP 8.3), and CRB/PowerTools
# (build -devel deps). No-op on Debian.
ensure_rhel_repos() {
  [[ "$OS_FAMILY" == rhel ]] || return 0
  local v="$OS_VERSION_MAJOR"
  if ! pkg_installed epel-release; then
    say "enabling EPEL"
    dnf install -y -q "https://dl.fedoraproject.org/pub/epel/epel-release-latest-${v}.noarch.rpm" \
      || dnf install -y -q epel-release \
      || warn "EPEL setup failed — certbot/whois/fail2ban may be unavailable"
  fi
  # CRB (RHEL 9) / PowerTools (older) — needed by several EPEL -devel deps.
  dnf config-manager --set-enabled crb        >/dev/null 2>&1 \
    || dnf config-manager --set-enabled powertools >/dev/null 2>&1 || true
  # Remi for PHP 8.3 (Vloud standardises on 8.3 across both families).
  if ! pkg_installed remi-release; then
    say "enabling Remi (PHP 8.3)"
    dnf install -y -q "https://rpms.remirepo.net/enterprise/remi-release-${v}.rpm" \
      || warn "Remi setup failed — PHP 8.3 may be unavailable"
  fi
  dnf module reset  -y php          >/dev/null 2>&1 || true
  dnf module enable -y "php:remi-8.3" >/dev/null 2>&1 \
    || warn "could not enable php:remi-8.3 dnf module"
}

# Best-effort open a host port. ufw on Debian, firewalld on RHEL. Only
# acts when the firewall is actually active so we never silently turn a
# firewall on under the operator.
fw_open_port() { # fw_open_port PORT [tcp|udp]
  local port="$1" proto="${2:-tcp}"
  case "$OS_FAMILY" in
    rhel)
      if command -v firewall-cmd >/dev/null 2>&1 && systemctl is-active --quiet firewalld; then
        firewall-cmd --permanent --add-port="${port}/${proto}" >/dev/null 2>&1 || true
      fi ;;
    debian)
      if command -v ufw >/dev/null 2>&1 && ufw status 2>/dev/null | grep "Status: active" >/dev/null; then
        ufw allow "${port}/${proto}" >/dev/null 2>&1 || true
      fi ;;
  esac
}

fw_reload() {
  if [[ "$OS_FAMILY" == rhel ]] && command -v firewall-cmd >/dev/null 2>&1 \
     && systemctl is-active --quiet firewalld; then
    firewall-cmd --reload >/dev/null 2>&1 || true
  fi
}

# Minimum SELinux config for the engine to start under enforcing mode:
# allow nginx → engine proxy, label the engine port, restore contexts on
# the state dir. No-op on Debian and when SELinux is disabled/absent.
configure_selinux() {
  [[ "$OS_FAMILY" == rhel ]] || return 0
  command -v getenforce >/dev/null 2>&1 || return 0
  local mode; mode="$(getenforce 2>/dev/null || echo Disabled)"
  [[ "$mode" == "Enforcing" || "$mode" == "Permissive" ]] || return 0
  say "configuring SELinux for the engine (current mode: $mode)"
  # nginx must be allowed to connect to the upstream engine socket.
  setsebool -P httpd_can_network_connect 1 2>/dev/null \
    || warn "setsebool httpd_can_network_connect failed"
  # Label the engine's loopback port as an http port so nginx can proxy.
  if command -v semanage >/dev/null 2>&1; then
    semanage port -a -t http_port_t -p tcp "$VLOUD_PORT" 2>/dev/null \
      || semanage port -m -t http_port_t -p tcp "$VLOUD_PORT" 2>/dev/null || true
  else
    warn "semanage missing (install policycoreutils-python-utils) — engine port may be blocked under enforcing SELinux"
  fi
  # Restore default contexts on the engine state tree.
  [[ -d /var/lib/vloud ]] && restorecon -Rv /var/lib/vloud >/dev/null 2>&1 || true
  ok "SELinux prepared (httpd_can_network_connect=on, port $VLOUD_PORT labelled http_port_t)"
}

# ─── Lifecycle progress JSONL emitter ───
#
# Append one JSON line per lifecycle event to VLOUD_PROGRESS_FILE.
# Schema (stable):
#
#   { "ts":  "<iso8601-z>",
#     "apply_run_id": <int|null>,
#     "phase": "upgrade" | "rollback" | "fresh" | "repair",
#     "stage": "<short-tag>",
#     "status": "started" | "ok" | "warn" | "failed",
#     "version_from": "<string|null>",
#     "version_to":   "<string|null>",
#     "msg": "<human-readable>" }
#
# Terminal markers emit with stage="terminal" so the engine can
# resolve apply_run state without parsing every intermediate line.
#
# - Append-only; never rewrites prior events
# - Rotates to ${VLOUD_PROGRESS_FILE}.<ts> when >RETAIN_BYTES
# - Stale rotated files older than STALE_DAYS are GC'd
# - Safe to call BEFORE the progress file directory exists (will
#   try to create it; failure is non-fatal — bootstrap.sh must
#   never abort because of progress logging)
# - Safe to call even when VLOUD_APPLY_RUN_ID is unset (engine
#   never reads those rows; operator-run upgrades still emit them
#   for the install.log forensics value)
emit_progress() {
  local stage="${1:-unknown}"
  local status="${2:-ok}"
  local msg="${3:-}"

  # Defensive: never let progress emission take down the script.
  set +e
  (
    set +e
    local dir; dir=$(dirname "$VLOUD_PROGRESS_FILE")
    [[ -d "$dir" ]] || install -d -m 0755 "$dir" 2>/dev/null

    # Rotate if oversize.
    if [[ -f "$VLOUD_PROGRESS_FILE" ]]; then
      local sz; sz=$(stat -c '%s' "$VLOUD_PROGRESS_FILE" 2>/dev/null || echo 0)
      if (( sz > VLOUD_PROGRESS_RETAIN_BYTES )); then
        local ts; ts=$(date -u +%Y%m%dT%H%M%SZ)
        mv "$VLOUD_PROGRESS_FILE" "${VLOUD_PROGRESS_FILE}.${ts}" 2>/dev/null
      fi
    fi

    # GC stale rotated files (best-effort).
    find "$(dirname "$VLOUD_PROGRESS_FILE")" -maxdepth 1 -type f \
        -name "$(basename "$VLOUD_PROGRESS_FILE").*" \
        -mtime "+${VLOUD_PROGRESS_STALE_DAYS}" -delete 2>/dev/null

    local ts_now; ts_now=$(date -u +%Y-%m-%dT%H:%M:%SZ)
    # Escape backslashes + quotes in msg (no jq dependency).
    local msg_esc="${msg//\\/\\\\}"
    msg_esc="${msg_esc//\"/\\\"}"
    # JSONify nullable apply_run_id (raw number when set, JSON null otherwise).
    local arid; arid="${VLOUD_APPLY_RUN_ID:-null}"
    [[ -z "$arid" ]] && arid=null
    [[ "$arid" != "null" ]] && ! [[ "$arid" =~ ^[0-9]+$ ]] && arid=null
    # Phase is derived from MODE; default 'upgrade' until MODE is set.
    local phase="${MODE:-upgrade}"
    local vfrom="${VLOUD_VERSION_FROM:-null}"
    local vto="${VLOUD_VERSION_TO:-null}"
    [[ "$vfrom" != "null" ]] && vfrom="\"$vfrom\""
    [[ "$vto"   != "null" ]] && vto="\"$vto\""

    printf '{"ts":"%s","apply_run_id":%s,"phase":"%s","stage":"%s","status":"%s","version_from":%s,"version_to":%s,"msg":"%s"}\n' \
      "$ts_now" "$arid" "$phase" "$stage" "$status" "$vfrom" "$vto" "$msg_esc" \
      >> "$VLOUD_PROGRESS_FILE" 2>/dev/null
  ) || true
  set -e
}

# Mark a terminal status for the current apply_run. Engine watchers
# resolve the run state from the most recent terminal event.
emit_progress_terminal() {
  local status="${1:-ok}"
  local msg="${2:-}"
  emit_progress terminal "$status" "$msg"
}

say "Vloud bootstrap starting on $PRETTY_NAME"
ok  "logging to $LOG_FILE"

# ─── Release-verification helpers (hoisted before mode dispatch) ───
#
# The build-release.sh pipeline substitutes the literal placeholder
# `__VLOUD_RELEASE_PUBKEY_PEM__` with the real PEM at release time. If
# the placeholder remains, $VLOUD_RELEASE_PUBKEY_PEM must be set OR
# VLOUD_INSECURE_SKIP_VERIFY=1 + VLOUD_TESTING=1 must both be set.
# These three primitives (constant + b64url_decode + verify_release_chain)
# are used by BOTH the fresh-install flow (phase 3 below) AND the
# upgrade flow (do_upgrade_flow above the dispatch). They must be
# defined here so both can call them.
# ─── corepack shims (hoisted before mode dispatch) ───
#
# REQUIRED for pipeline builds, not a nicety.
#
# Node ships corepack, but without `corepack enable` the only way to reach pnpm
# or yarn is `corepack pnpm …`. A pipeline step can be written that way, but the
# package.json scripts it then runs cannot: a script like
#
#     "build": "next build && pnpm run postbuild:standalone"
#
# invokes BARE `pnpm`, which is not on PATH, and the build dies at the second
# command with `sh: 1: pnpm: not found` — after a full successful compile, so it
# reads as a build failure rather than a missing binary.
#
# Called from ALL THREE flows. Fresh install is not enough: this is host-level
# setup outside $VLOUD_INSTALL_DIR, so an existing host that upgrades into a
# release with this fix would otherwise never get the shim, and --repair is
# where an operator goes to fix exactly this class of thing.
#
# Best-effort by design: a host whose corepack cannot write its shim directory
# still installs and still runs Vloud — it just cannot run pnpm/yarn package
# scripts, which is worth a warning and not a refusal.
ensure_corepack_shims() {
  if ! command -v corepack >/dev/null 2>&1; then
    warn "corepack is not present in this Node install — pnpm/yarn package scripts will not resolve"
    return 0
  fi
  # --install-directory keeps the shims on the default PATH of a pipeline step,
  # which does not inherit a login shell's PATH. Fall back to plain enable for
  # Node builds that reject the flag.
  if corepack enable --install-directory /usr/local/bin >/dev/null 2>&1 \
     || corepack enable >/dev/null 2>&1; then
    if command -v pnpm >/dev/null 2>&1; then
      ok "corepack shims enabled (pnpm on PATH)"
    else
      ok "corepack shims enabled"
    fi
  else
    warn "corepack enable failed — package scripts that call bare 'pnpm' or 'yarn' will not resolve"
  fi
  return 0
}

# ─── NodeSource 20.x (hoisted before mode dispatch) ───
#
# Point apt at Node 20 and make sure it actually WINS. Both halves matter, and
# the second is where this used to fail silently.
#
# ## Why an origin pin is not enough
#
# The old code wrote the 20.x repo and then pinned:
#
#     Package: nodejs
#     Pin: origin deb.nodesource.com
#     Pin-Priority: 1001
#
# On a host that already had NodeSource configured for a DIFFERENT major, that
# pin cannot do its job. Both repos share the origin `deb.nodesource.com`, so
# the pin matches BOTH equally and apt falls back to its normal rule: install
# the highest version available. That is 22.x, every time.
#
# Worse, the pre-existing repo is usually in the newer deb822 format
# (`nodesource.sources`), while we write the one-line `nodesource.list`. They
# are different files, so writing ours did not replace theirs — it just added a
# second NodeSource repo alongside it. Observed on a real host: our
# `nodesource.list` said node_20.x, and `apt-cache madison nodejs` offered
# nothing but 22.x.
#
# So: disable every other NodeSource source first, and pin by VERSION, which is
# the only expression of "20, not 22" that survives two same-origin repos.
configure_nodesource20() {
  install -d -m 0755 /etc/apt/keyrings
  curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key \
    | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg --yes 2>/dev/null

  # Move aside any OTHER NodeSource source, whatever its filename or format.
  # Renamed rather than deleted: it is the operator's file, and .disabled is
  # both inert to apt and trivially reversible.
  local f
  for f in /etc/apt/sources.list.d/*; do
    [[ -f "$f" ]] || continue
    [[ "$f" == /etc/apt/sources.list.d/nodesource.list ]] && continue
    [[ "$f" == *.disabled-by-vloud ]] && continue
    if grep -qs "deb.nodesource.com" "$f" && ! grep -qs "node_20\.x" "$f"; then
      mv -f "$f" "$f.disabled-by-vloud"
      warn "disabled competing NodeSource repo $(basename "$f") (it offered a Node major other than 20)"
    fi
  done

  echo 'deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_20.x nodistro main' \
    > /etc/apt/sources.list.d/nodesource.list

  # Pin by VERSION, not origin — see the note above. Priority >1000 is the only
  # band that lets apt install a LOWER version than the one already present,
  # which is exactly what repairing a Node 22 host requires.
  #
  # THE FILENAME IS PART OF THE FIX. apt reads /etc/apt/preferences.d in
  # alphabetical order and applies the FIRST entry that matches a package —
  # not the highest priority among all matches. NodeSource's own installer
  # ships /etc/apt/preferences.d/nodejs:
  #
  #     Package: nodejs
  #     Pin: origin deb.nodesource.com
  #     Pin-Priority: 600
  #
  # That matches our node_20.x repo too (same origin), and "nodejs" sorts
  # before "vloud-nodesource", so it won and our 1001 was never applied.
  # Measured on a real host: the pin file was present and correct, and
  # `apt-cache policy nodejs` still reported 600 for 20.20.2 — not enough to
  # downgrade from 22, so the install aborted. Renaming the file to sort first
  # changed the same host to "1 downgraded / Inst nodejs (20.20.2)".
  rm -f /etc/apt/preferences.d/vloud-nodesource   # pre-0.9.41 name, sorted too late
  cat > /etc/apt/preferences.d/00-vloud-nodesource <<'PREF'
Package: nodejs
Pin: version 20.*
Pin-Priority: 1001
PREF
  # NOT -qq. On agent-work-01 this step silently failed to fetch the node_20.x
  # index — /var/lib/apt/lists/ had no node_20 entry afterwards — and because
  # the error was suppressed, the install aborted a moment later saying Node 20
  # was unavailable, with nothing anywhere explaining why. A refresh that fails
  # must say so.
  if ! apt-get update -o Dpkg::Use-Pty=0 >/tmp/vloud-apt-update.log 2>&1; then
    warn "apt-get update reported errors while configuring the Node 20 repo:"
    grep -iE '^(E|W):' /tmp/vloud-apt-update.log | head -5 | while IFS= read -r l; do
      warn "  $l"
    done
  fi
  # Say plainly whether the repo we just configured is actually reachable,
  # because everything downstream depends on it.
  if ! apt-cache madison nodejs 2>/dev/null | grep -E '\|\s*20\.' >/dev/null; then
    warn "the node_20.x repository is configured but apt still sees no 20.x package"
    grep -iE 'nodesource' /tmp/vloud-apt-update.log | head -3 | while IFS= read -r l; do
      warn "  $l"
    done
  fi
}

# Can this host actually GET Node 20?
#
# Called in preflight, because the answer decides whether the install may
# proceed AT ALL — and the old code asked it far too late. The Node assertion
# used to run after the package phase had already installed and restarted the
# whole LAMP stack, so a host that could not reach Node 20 was left with its
# services restarted, its packages swapped, and no Vloud: all of the damage,
# none of the benefit. Ask first, break nothing.
#
# Returns 0 when Node 20 is already installed or is offered by apt.
node20_obtainable() {
  local cur
  if cur=$(node --version 2>/dev/null) && [[ "${cur#v}" == 20.* ]]; then
    return 0
  fi
  [[ "$OS_FAMILY" == debian ]] || return 0   # RHEL path handles its own module
  configure_nodesource20
  apt-cache madison nodejs 2>/dev/null | grep -E '\|\s*20\.' >/dev/null
}

# ─── Protecting what the host already runs (hoisted before mode dispatch) ───

# Is there already a database server here?
#
# Named, explicit and separate from the generic never-remove rule in
# install_packages, because this is the specific outcome that hurt: apt
# resolved `mariadb-server`'s Conflicts: by DELETING MySQL 8.0 and standing
# MariaDB up on an empty datadir. The generic rule prevents that now, but a
# database server is worth refusing by name rather than by side effect — and
# saying so in words the operator can act on.
existing_database_server() {
  local u id
  for u in mysql mysqld mariadb; do
    if systemctl is-active --quiet "$u.service" 2>/dev/null; then
      # Resolve the alias. On a MariaDB host mysql.service is an ALIAS for
      # mariadb.service and reports active, so naming the probe rather than the
      # unit tells the operator their box runs "mysql" when it runs MariaDB —
      # exactly the confusion this whole change exists to prevent.
      id=$(systemctl show -p Id --value "$u.service" 2>/dev/null)
      echo "${id:-$u.service}"
      return 0
    fi
  done
  for u in mysql-server mysql-server-8.0 mariadb-server percona-server-server; do
    if pkg_installed "$u"; then echo "$u"; return 0; fi
  done
  return 1
}

# Remove our preferred DB package from a package list when the host already has
# a database server. Echoes the filtered list.
drop_db_if_present() {
  local existing p out=()
  if ! existing=$(existing_database_server); then printf '%s\n' "$@"; return 0; fi
  for p in "$@"; do
    case "$p" in
      mariadb-server|mysql-server|postgresql-server)
        # >&2 is load-bearing: this function returns the filtered PACKAGE LIST
        # on stdout, and warn() prints to stdout. Without the redirect the
        # warning text is captured as part of the list and handed to apt as
        # package names.
        warn "not installing '$p': this host already runs a database server ($existing)" >&2
        warn "  Vloud will use the existing server. Nothing is removed, replaced or reinitialised." >&2
        ;;
      *) out+=("$p") ;;
    esac
  done
  printf '%s\n' ${out[@]+"${out[@]}"}
}

# ── Service-integrity net ────────────────────────────────────────────────────
#
# The installer must not take down anything it did not install. It has no code
# that stops a shared service — nginx is only ever `reload`ed, and only after
# `nginx -t` passes — yet a live site still went dark for 14 minutes, because
# apt reconfiguring an already-installed nginx restarted it into a port clash
# it never recovered from (no Restart= policy, so systemd gave up after one
# try).
#
# Being right in principle was not enough, so this verifies it in practice:
# snapshot what is running before, check it after, and put back anything that
# stopped. An installer that knocks over a production service and does not
# notice is the failure mode; noticing and fixing it is cheap.
VLOUD_SERVICES_BEFORE=""

snapshot_running_services() {
  VLOUD_SERVICES_BEFORE=$(systemctl list-units --type=service --state=running \
    --no-legend --no-pager 2>/dev/null | awk '{print $1}' | grep -v '^vloud' | sort)
}

verify_services_survived() {
  [[ -n "$VLOUD_SERVICES_BEFORE" ]] || return 0
  local now casualties u
  now=$(systemctl list-units --type=service --state=running \
    --no-legend --no-pager 2>/dev/null | awk '{print $1}' | sort)
  casualties=$(comm -23 <(printf '%s\n' "$VLOUD_SERVICES_BEFORE") <(printf '%s\n' "$now"))
  [[ -n "$casualties" ]] || { ok "every service that was running before the install is still running"; return 0; }

  for u in $casualties; do
    # A unit in `activating` is mid-restart, not down. Some hosts carry units
    # that crash-loop permanently (observed: two at ~1.2M restarts), and those
    # flip in and out of `running` constantly — reporting them as casualties of
    # OUR install is noise, and noise is how a real casualty gets ignored.
    # Give it a moment and re-check before saying anything.
    sleep 1
    case "$(systemctl is-active "$u" 2>/dev/null)" in
      active|activating|reloading) continue ;;
    esac
    fail "$u was RUNNING before this install and is not running now — restarting it"
    systemctl reset-failed "$u" 2>/dev/null || true
    if systemctl start "$u" 2>/dev/null && systemctl is-active --quiet "$u" 2>/dev/null; then
      ok "  restarted $u"
    else
      fail "  could NOT restart $u — investigate with 'journalctl -u $u -n 50'"
    fi
  done
}

EMBEDDED_RELEASE_PUBKEY_PEM='-----BEGIN PUBLIC KEY-----
MCowBQYDK2VwAyEAWkdy6RSETT2bSnm7T1fRDe8sr7keuU5EO/Wvk2gFXSM=
-----END PUBLIC KEY-----'

b64url_decode() {
  local s="$1"
  s="${s//-/+}"; s="${s//_//}"
  local pad=$(( (4 - ${#s} % 4) % 4 ))
  while (( pad-- > 0 )); do s+="="; done
  printf '%s' "$s" | base64 -d
}

verify_release_chain() {
  local tarball="$1" checksum="$2" signature="$3"
  local expected_artifact="${4:-$(basename "$tarball")}"
  local pubkey_pem_data
  if [[ -n "${VLOUD_RELEASE_PUBKEY_PEM:-}" ]]; then
    pubkey_pem_data="$VLOUD_RELEASE_PUBKEY_PEM"
  elif [[ "$EMBEDDED_RELEASE_PUBKEY_PEM" != "__VLOUD_RELEASE_PUBKEY_PEM__" ]]; then
    pubkey_pem_data="$EMBEDDED_RELEASE_PUBKEY_PEM"
  else
    if [[ "${VLOUD_INSECURE_SKIP_VERIFY:-0}" == "1" && "${VLOUD_TESTING:-0}" == "1" ]]; then
      warn "VERIFICATION SKIPPED — VLOUD_INSECURE_SKIP_VERIFY=1 set (test mode)"
      return 0
    fi
    die "release public key not configured.  Set VLOUD_RELEASE_PUBKEY_PEM or use a release-built bootstrap.sh.  To skip for testing: VLOUD_INSECURE_SKIP_VERIFY=1 VLOUD_TESTING=1."
  fi
  local pubkey_file
  pubkey_file=$(mktemp /tmp/vloud-release-key-XXXXXX.pem)
  printf '%s' "$pubkey_pem_data" > "$pubkey_file"

  if ! ( cd "$(dirname "$tarball")" && sha256sum -c "$(basename "$checksum")" >/dev/null 2>&1 ); then
    rm -f "$pubkey_file"
    die "release tarball sha256 MISMATCH — refusing install"
  fi

  local jws
  jws=$(<"$signature")
  jws="${jws//[$'\n\r ']/}"
  IFS='.' read -r jws_header jws_payload jws_sig <<<"$jws"
  if [[ -z "$jws_header" || -z "$jws_payload" || -z "$jws_sig" ]]; then
    rm -f "$pubkey_file"
    die "release signature is not a JWS compact serialization — refusing install"
  fi

  local header_json alg typ kid
  header_json=$(b64url_decode "$jws_header")
  alg=$(printf '%s' "$header_json" | sed -n 's/.*"alg":"\([^"]*\)".*/\1/p')
  typ=$(printf '%s' "$header_json" | sed -n 's/.*"typ":"\([^"]*\)".*/\1/p')
  kid=$(printf '%s' "$header_json" | sed -n 's/.*"kid":"\([^"]*\)".*/\1/p')
  if [[ "$alg" != "EdDSA" || "$typ" != "vloud-release+jws" ]]; then
    rm -f "$pubkey_file"
    die "release signature header invalid (alg=$alg typ=$typ) — refusing install"
  fi
  if [[ "$kid" != "release-v1" && "$kid" != "release-v2" ]]; then
    rm -f "$pubkey_file"
    die "release signature kid '$kid' not in allowed set — refusing install"
  fi

  local sig_bin sign_input
  sig_bin=$(mktemp /tmp/vloud-sig-XXXXXX.bin)
  b64url_decode "$jws_sig" > "$sig_bin"
  sign_input=$(mktemp /tmp/vloud-signin-XXXXXX)
  printf '%s' "$jws_header.$jws_payload" > "$sign_input"
  if ! openssl pkeyutl -verify -pubin -inkey "$pubkey_file" \
       -rawin -in "$sign_input" -sigfile "$sig_bin" >/dev/null 2>&1; then
    rm -f "$pubkey_file" "$sig_bin" "$sign_input"
    die "release signature INVALID — refusing install"
  fi
  rm -f "$sig_bin" "$sign_input"

  local payload_json payload_sha payload_artifact actual_sha
  payload_json=$(b64url_decode "$jws_payload")
  payload_sha=$(printf '%s' "$payload_json" | sed -n 's/.*"sha256":"\([0-9a-f]*\)".*/\1/p')
  payload_artifact=$(printf '%s' "$payload_json" | sed -n 's/.*"artifact":"\([^"]*\)".*/\1/p')
  actual_sha=$(sha256sum "$tarball" | awk '{print $1}')
  if [[ "$payload_sha" != "$actual_sha" ]]; then
    rm -f "$pubkey_file"
    die "release signed sha256 ($payload_sha) does not match tarball ($actual_sha) — refusing install"
  fi
  if [[ "$payload_artifact" != "$expected_artifact" ]]; then
    rm -f "$pubkey_file"
    die "release signed artifact ($payload_artifact) does not match expected artifact ($expected_artifact) — refusing install"
  fi

  rm -f "$pubkey_file"
  ok "release verified: $expected_artifact (kid=$kid)"
}

# ─── Mode detection + dispatch ───
#
# This block decides what kind of install we're running. After this,
# the script either dispatches to a mode-specific function (and exits)
# or falls through to the linear fresh-install phases 1-6 below.
MODE="${VLOUD_INSTALL_MODE:-}"
# Opt-in runtimes. Empty unless the operator asked for them.
VLOUD_WITH_DOTNET="${VLOUD_WITH_DOTNET:-0}"
for arg in "$@"; do
  case "$arg" in
    --upgrade)              MODE=upgrade ;;
    --rollback)             MODE=rollback ;;
    --repair)               MODE=repair ;;
    --doctor|--health)      MODE=doctor ;;
    --force-clean-install)  MODE=force-clean ;;
    --uninstall)            MODE=uninstall ;;
    --yes|-y)               VLOUD_ASSUME_YES=1 ;;
    --fresh)                MODE=fresh ;;
    # OPTIONAL .NET runtime for VloudDeploy@1 `runtime: dotnet` apps.
    #
    # Opt-in on purpose: the ASP.NET Core runtime is a few hundred megabytes,
    # most hosts will never run a .NET app, and the engine NEVER installs a
    # runtime implicitly during a deploy — a deploy that apt-installs an SDK on
    # a shared host is a surprise, not a feature. A dotnet deploy on a host
    # without it fails with an actionable message naming this flag.
    --with-dotnet)          VLOUD_WITH_DOTNET=1 ;;
    # PR1: per-version targeting. When absent, behaviour is unchanged
    # (resolve to install.vloud.app/releases/latest/…).
    --version=*)            VLOUD_TARGET_VERSION="${arg#--version=}" ;;
    --version)              shift ; VLOUD_TARGET_VERSION="${1:-}" ; continue ;;
    # PR1: rollback target. When set with MODE=rollback, restore that
    # specific .pre-upgrade-* slot; otherwise restore the newest.
    --to=*)                 VLOUD_ROLLBACK_TO="${arg#--to=}" ;;
    --to)                   shift ; VLOUD_ROLLBACK_TO="${1:-}" ; continue ;;
    # PR1: engine apply-run correlation. Tags every progress event
    # so the engine can match JSONL rows to its apply-run row.
    --apply-run-id=*)       VLOUD_APPLY_RUN_ID="${arg#--apply-run-id=}" ;;
    --apply-run-id)         shift ; VLOUD_APPLY_RUN_ID="${1:-}" ; continue ;;
    # Phase E (2026-05-14): explicit "this rollback was fired by
    # systemd, not by an operator click". Accepted but currently
    # behaves identically to plain --rollback. Mostly here so the
    # systemd unit's ExecStart doesn't get rejected as unknown args.
    --automatic)            VLOUD_ROLLBACK_AUTOMATIC=1 ;;
    --help|-h)
      sed -n '2,43p' "$0" | sed 's/^# \{0,1\}//'
      exit 0
      ;;
    *)
      # Ignore unknown args here; phase scripts may consume their own.
      ;;
  esac
done

# PR1: when --version=<v> is passed, override the default URL with
# the per-version path. Operators who already set VLOUD_RELEASE_URL
# explicitly (private mirror / offline-mirror customers) win.
#
# The release blob stores per-version artifacts under a "v"-prefixed
# directory (releases/v<version>/…) — matching the git tag (v*) the CI
# publishes from and the signed manifest's `image` URL. The tarball
# FILE itself is unprefixed (vloud-<version>.tar.gz). Without the "v"
# on the directory the versioned URL 404s and every fleet self-update
# dies at download (only the unversioned latest/ path worked).
if [[ -n "$VLOUD_TARGET_VERSION" ]] && [[ "$VLOUD_RELEASE_URL" == "https://install.vloud.app/releases/latest/vloud-latest.tar.gz" ]]; then
  VLOUD_RELEASE_URL="https://install.vloud.app/releases/v${VLOUD_TARGET_VERSION}/vloud-${VLOUD_TARGET_VERSION}.tar.gz"
fi

# Auto-detect if no explicit mode.
if [[ -z "$MODE" ]]; then
  if [[ -d "$VLOUD_INSTALL_DIR/packages/server/dist" ]]; then
    MODE=upgrade
    say "existing install detected at $VLOUD_INSTALL_DIR — auto-mode → UPGRADE"
    warn "running upgrade (preserves DB + config). Use --force-clean-install to wipe."
  else
    MODE=fresh
    say "no existing install — auto-mode → FRESH INSTALL"
  fi
fi

# ── mode helpers (defined here so dispatch below can use them) ──

# Backup critical state to /var/lib/vloud/backups/<label>-<ts>.tar.gz
# AND copy /etc/vloud.env. Idempotent + non-destructive.
do_backup_state() {
  local label="${1:-pre-upgrade}"
  local ts; ts=$(date -u +%Y%m%dT%H%M%SZ)
  local backup_dir=/var/lib/vloud/backups
  mkdir -p "$backup_dir"
  local backup="$backup_dir/$label-$ts.tar.gz"

  # Progress goes to STDERR, deliberately.
  #
  # This function returns the archive path on stdout, so every caller writes
  # `do_backup_state ... >/dev/null` — which silenced the progress lines too.
  # The result, measured on a live host: seven minutes of complete silence
  # immediately after "PostgreSQL ready", with the operator reasonably
  # concluding the upgrade had hung. Interrupting an upgrade at this point is
  # genuinely dangerous, so the silence was the bug, not the duration.
  say "backing up critical state → $backup" >&2
  say "this archives the engine's state tree and can take a few minutes on a host with large apps" >&2
  emit_progress backup started "$backup"

  local sources=()
  [[ -f $ENV_FILE ]]                                    && sources+=("$ENV_FILE")
  [[ -d /etc/vloud ]]                                   && sources+=("/etc/vloud")
  [[ -f $VLOUD_INSTALL_DIR/packages/server/vloud.db ]]  && sources+=("$VLOUD_INSTALL_DIR/packages/server/vloud.db")
  [[ -d /var/lib/vloud ]]                               && sources+=("/var/lib/vloud")
  # nginx vhosts the engine generated (vloud-*.conf in sites-enabled, excluding system app names)
  while IFS= read -r f; do sources+=("$f"); done < <(
    ls /etc/nginx/sites-enabled/vloud-*.conf 2>/dev/null |
    grep -vE '/vloud-(app-|admin|commercial-|marketing|portal|license)' || true
  )

  if [[ ${#sources[@]} -gt 0 ]]; then
    # CRITICAL: exclude the backups dir itself. /var/lib/vloud is a
    # source, and it CONTAINS $backup_dir — without this exclude each
    # new pre-upgrade backup swallows every previous one, so the
    # archives grow exponentially (observed 7M → 24M → … → 8.2G) and
    # eventually fill the disk, which then fails the very next upgrade
    # at the backup stage ("N MB free on /opt"). Also skip large,
    # regenerable/live runtime data that has no business in a config +
    # DB state snapshot (mongodb is a live DB; db-snapshots are derived).
    #
    # node_modules and artifacts are excluded for the same reason, and it
    # is not a small saving: measured on the reference host, node_modules
    # accounted for 5.4 GB of a 6.1 GB apps tree and artifacts for a further
    # 5.0 GB — the bulk of a 2.5 GB archive and most of its seven minutes.
    # Both are rebuilt by a redeploy, and neither is touched by an engine
    # upgrade: the swap replaces /opt/vloud, while this tree stays put. So
    # archiving them bought nothing and cost the whole upgrade window.
    local excludes=(
      --exclude="$backup_dir"
      --exclude=/var/lib/vloud/mongodb
      --exclude=/var/lib/vloud/db-snapshots
      --exclude='node_modules'
      --exclude=/var/lib/vloud/artifacts
    )

    # gzip is single-threaded. pigz is already a dependency on these hosts
    # and uses every core, which on a 12-core box is the difference between
    # minutes and seconds for the same archive.
    local compressor=(gzip)
    if command -v pigz >/dev/null 2>&1; then compressor=(pigz); fi

    local started; started=$(date +%s)
    if tar -cf - --ignore-failed-read "${excludes[@]}" "${sources[@]}" 2>/dev/null \
         | "${compressor[@]}" > "$backup"; then
      :
    else
      warn "backup partial — some files unreadable" >&2
    fi
    local elapsed=$(( $(date +%s) - started ))
    ok "backup: $backup ($(du -h "$backup" 2>/dev/null | awk '{print $1}')) in ${elapsed}s" >&2
    emit_progress backup ok "$backup"

    _prune_state_backups "$backup_dir" "$label"
  else
    warn "nothing to back up (no existing state)" >&2
  fi

  echo "$backup"
}

# Keep the newest few state archives and delete the rest.
#
# Nothing ever removed these. Measured on the reference host: 19 GB of
# pre-upgrade tarballs at 2.5 GB each, on a disk that was 91% full — every
# upgrade added another and brought the next one closer to failing at the
# "needs ~1.5 GB for the staging tree" guard. Retaining a couple of
# generations is the point of the backup; retaining all of them is what
# eventually prevents an upgrade entirely.
_prune_state_backups() {
  local dir="$1" label="$2" keep="${VLOUD_KEEP_STATE_BACKUPS:-3}"
  [[ -d "$dir" ]] || return 0

  local old
  # Newest first, drop the ones we keep, delete the remainder. Sorted by
  # name, which is safe because the timestamp is a fixed-width UTC stamp.
  old=$(ls -1 "$dir"/"$label"-*.tar.gz 2>/dev/null | sort -r | tail -n +$((keep + 1)))
  [[ -n "$old" ]] || return 0

  local freed=0 f
  while IFS= read -r f; do
    [[ -f "$f" ]] || continue
    freed=$(( freed + $(du -m "$f" 2>/dev/null | awk '{print $1}') ))
    rm -f "$f"
  done <<< "$old"

  [[ $freed -gt 0 ]] && ok "pruned older $label archives, freed ${freed} MB (kept newest $keep)" >&2
  return 0
}

# Stop vloud services gracefully. Idempotent.
#
# NOTE: the engine spawns THIS bootstrap as a child of vloud.service.
# Surviving the `systemctl stop vloud.service` below is handled by the
# unit's KillMode=process (see the [Service] block written in
# do_emit_systemd_units) — systemd then signals only the engine's main
# PID on stop, never the whole control group, so this upgrade process
# lives on to finish the swap and restart. Do NOT revert KillMode to
# the default control-group or engine-driven upgrades will kill
# themselves here (dashboard stuck "Installing…").
do_stop_services() {
  say "stopping vloud services"
  for u in vloud.service vloud-job-worker.service vloud-scheduler.service; do
    if systemctl is-active --quiet "$u" 2>/dev/null; then
      systemctl stop "$u" 2>/dev/null && ok "stopped $u" || warn "could not stop $u"
    fi
  done
}

# Start vloud services. Reports health within 30s.
#
# Critical: `systemctl reset-failed` BEFORE each start. Without this,
# a previous failed-start cascade (e.g. the new engine crash-looped
# during an upgrade attempt) leaves the unit in `start-limit-hit`.
# systemd then refuses every subsequent start with "Start request
# repeated too quickly" — including the rollback restart, which is
# exactly the cascading-failure mode that bit the 0.5.1-beta upgrade
# on 188.245.113.223 (2026-05-13).
#
# We also no longer swallow stderr on `systemctl start` — the actual
# systemd refusal message is the single most useful piece of
# forensics when a start fails, and `2>/dev/null` was hiding it.
do_start_services_with_healthcheck() {
  say "starting vloud services"
  systemctl reset-failed vloud.service vloud-job-worker.service vloud-scheduler.service 2>/dev/null || true
  systemctl daemon-reload
  systemctl start vloud.service             || warn "vloud.service start returned $?"
  systemctl start vloud-job-worker.service  || warn "vloud-job-worker.service start returned $?"
  systemctl start vloud-scheduler.service   || warn "vloud-scheduler.service start returned $?"
  local i rc
  for i in {1..15}; do
    rc=$(curl -s -o /dev/null -w '%{http_code}' --max-time 2 "http://127.0.0.1:$VLOUD_PORT/api/health" 2>/dev/null || echo 000)
    if [[ "$rc" == "200" ]]; then
      ok "engine health-check passed in ${i}s"
      return 0
    fi
    sleep 2
  done
  fail "engine did not pass health-check within 30s — see journalctl -u vloud"
  return 1
}

# Dump the last lines of vloud.service's journal so operators have
# forensics in the install log when a start failed. Cheap; only runs
# on failure paths.
do_dump_failure_journal() {
  say "vloud.service journal (last 40 lines) — for forensics:"
  journalctl -u vloud.service -n 40 --no-pager 2>&1 | sed 's/^/  | /'
}

# Per-file integrity verification (PR1).
#
# The release tarball SHA is already covered by the signed manifest
# (verify_release_chain). This adds a defence-in-depth check: if the
# release build pipeline shipped a dist/SHA256SUMS file, verify every
# extracted file against it. Catches the (uncommon but plausible)
# case where the tarball decompressed but a single file was corrupted
# OR the build pipeline shipped an inconsistent tarball.
#
# Missing SHA256SUMS is NOT fatal — older releases don't ship it,
# and offline/private-mirror customers may strip it. Operator gets
# a `warn` instead of an abort.
do_verify_sha256sums() {
  local staging_root="$1"
  local checked=0 missing=0
  for sums in \
      "$staging_root/packages/server/dist/SHA256SUMS" \
      "$staging_root/packages/web/dist/SHA256SUMS"; do
    if [[ -f "$sums" ]]; then
      local sums_dir; sums_dir=$(dirname "$sums")
      if ( cd "$sums_dir" && sha256sum -c --quiet --ignore-missing SHA256SUMS >/dev/null 2>&1 ); then
        ok "per-file integrity OK: $sums"
        checked=$((checked+1))
      else
        emit_progress verify-sha256sums failed "mismatch in $sums"
        die "per-file integrity FAILED in $sums_dir — refusing swap" 9
      fi
    else
      missing=$((missing+1))
    fi
  done
  if (( checked == 0 )) && (( missing > 0 )); then
    warn "release ships no SHA256SUMS — per-file integrity SKIPPED (manifest signature still in force)"
    emit_progress verify-sha256sums warn "no SHA256SUMS in tarball"
  else
    emit_progress verify-sha256sums ok "verified=$checked"
  fi
}

# ABI preflight: confirm the staging tree's native modules can load
# against the host's Node ABI. If the release-runtime.json's
# node_module_version differs from the host's, attempt `npm rebuild`
# inside the staging tree (better-sqlite3's prebuild-install will
# fetch a binary matching the host's Node). Only proceed if the
# rebuild succeeds AND a post-rebuild require() of better-sqlite3
# loads cleanly.
#
# Called from do_upgrade_flow BEFORE the atomic swap so a mismatch
# never reaches the running system.
do_abi_preflight_or_rebuild() {
  local staging="$1"
  local runtime_json="$staging/release-runtime.json"
  local local_nmv
  local_nmv=$(node -e 'process.stdout.write(String(process.versions.modules))')

  if [[ ! -f "$runtime_json" ]]; then
    warn "release missing release-runtime.json — older tarball, doing best-effort rebuild"
  else
    local expected_nmv
    expected_nmv=$(sed -n 's/.*"node_module_version":[[:space:]]*\([0-9]*\).*/\1/p' "$runtime_json" | head -1)
    if [[ -n "$expected_nmv" && "$expected_nmv" == "$local_nmv" ]]; then
      ok "ABI match: tarball NMV=$expected_nmv = host NMV=$local_nmv"
      # Still run a load-test below; a matching NMV is necessary but
      # not sufficient (e.g. glibc / arch could still bite).
    else
      warn "ABI mismatch: tarball NMV=${expected_nmv:-?} host NMV=$local_nmv — rebuilding native modules in staging"
    fi
  fi

  # Rebuild every native module in staging against the host's Node.
  # better-sqlite3 + bcrypt are the canonical ones; sharp is optional.
  # We run npm rebuild over all of them and let npm decide which need
  # work. Quiet output unless something fails.
  say "rebuilding native bindings against host Node $(node --version) (NMV=$local_nmv)"
  if ! ( cd "$staging/packages/server" && \
         npm rebuild --no-audit --no-fund 2>&1 ) | tail -10 | sed 's/^/  | /'; then
    fail "npm rebuild failed in staging — see lines above"
    return 1
  fi

  # Post-rebuild verification: load better-sqlite3 in the staging tree
  # and open an in-memory DB. If the .node file is still ABI-wrong,
  # this throws and we return 1 — caller aborts before swap.
  if ! ( cd "$staging/packages/server" && \
         node -e '
           try {
             const Db = require("better-sqlite3");
             const db = new Db(":memory:");
             db.prepare("SELECT 1").get();
             db.close();
             process.stdout.write("ok");
           } catch (e) {
             process.stderr.write("LOAD_FAILED: " + e.message + "\n");
             process.exit(1);
           }
         ' >/dev/null ); then
    fail "better-sqlite3 still fails to load after rebuild — release is incompatible with this host"
    return 1
  fi
  ok "ABI preflight passed — native modules load cleanly against host Node"
  return 0
}

# ── --doctor ──────────────────────────────────────────────────
do_doctor_flow() {
  say "DOCTOR: read-only health check"
  echo
  printf "  %-22s active=%s enabled=%s\n" "vloud"            "$(systemctl is-active vloud 2>/dev/null)"            "$(systemctl is-enabled vloud 2>/dev/null)"
  printf "  %-22s active=%s enabled=%s\n" "vloud-job-worker" "$(systemctl is-active vloud-job-worker 2>/dev/null)" "$(systemctl is-enabled vloud-job-worker 2>/dev/null)"
  printf "  %-22s active=%s enabled=%s\n" "vloud-scheduler"  "$(systemctl is-active vloud-scheduler 2>/dev/null)"  "$(systemctl is-enabled vloud-scheduler 2>/dev/null)"
  printf "  %-22s active=%s\n"             "nginx"           "$(systemctl is-active nginx 2>/dev/null)"
  printf "  %-22s active=%s\n"             "named"           "$(systemctl is-active named 2>/dev/null)"
  echo
  say "port 2544 listener"
  ss -lntp 2>/dev/null | grep ':2544\b' | head -3 || warn "no listener on :2544"
  echo
  say "engine health"
  local health
  health=$(curl -s --max-time 4 "http://127.0.0.1:$VLOUD_PORT/api/health" 2>&1 || echo "(timeout)")
  echo "  $health"
  echo
  say "engine version"
  curl -s --max-time 4 "http://127.0.0.1:$VLOUD_PORT/api/system/version" 2>&1 | head -c 400; echo
  echo
  say "DB integrity"
  if [[ -f $VLOUD_INSTALL_DIR/packages/server/vloud.db ]]; then
    if command -v sqlite3 >/dev/null 2>&1; then
      local r; r=$(sqlite3 "$VLOUD_INSTALL_DIR/packages/server/vloud.db" 'PRAGMA integrity_check' 2>&1 | head -1)
      echo "  $r"
    else
      warn "sqlite3 not installed"
    fi
  else
    warn "DB not found at $VLOUD_INSTALL_DIR/packages/server/vloud.db"
  fi
  echo
  say "disk usage"
  df -h "$VLOUD_INSTALL_DIR" / 2>/dev/null | awk 'NR==1 || /\// {print "  "$0}' | head -3
  echo
  ok "DOCTOR done — no changes made"
}

# ── --repair ──────────────────────────────────────────────────
do_repair_flow() {
  say "REPAIR: re-emit systemd units + restart"
  if [[ ! -d "$VLOUD_INSTALL_DIR/packages/server/dist" ]]; then
    die "no install at $VLOUD_INSTALL_DIR — repair needs an existing install; run --fresh first" 3
  fi

  # Repair is the operator's "make this host correct again" button, so
  # it reinstates missing runtime packages too — not just systemd units.
  # Idempotent; no state is touched.
  ensure_system_packages

  # Re-emit systemd units from the install tree. The packaged units
  # live under $VLOUD_INSTALL_DIR/packaging/systemd/.
  local pkg_dir=$VLOUD_INSTALL_DIR/packaging/systemd
  if [[ -d $pkg_dir ]]; then
    for u in vloud.service vloud-job-worker.service vloud-scheduler.service vloud.slice; do
      if [[ -f "$pkg_dir/$u" ]]; then
        cp -f "$pkg_dir/$u" "/etc/systemd/system/$u"
        ok "wrote /etc/systemd/system/$u"
      fi
    done
  else
    warn "packaging/systemd/ not found in install tree — using existing unit files"
  fi
  # 0.6.0b — repair flow now also reconciles env-file + license-
  # pubkey + nginx default vhost. Repair is the operator's "make it
  # right again" lever; previously it only re-emitted systemd units,
  # which left the broken state from a partial 0.6.0 → 0.6.0a
  # upgrade in place.
  if ! do_reconcile_env_file 1; then
    die "env-file reconcile failed — refusing to restart services in known-broken state" 7
  fi

  # host-level build tooling — see ensure_corepack_shims.
  ensure_corepack_shims

  # license pubkey (mirrors do_upgrade_flow)
  if [[ -n "${VLOUD_LICENSE_SERVER_URL_DEFAULT:-}" ]]; then
    install -d -m 0755 /etc/vloud
    TMP_PEM=$(mktemp /tmp/vloud-license-pubkey.XXXXXX.pem)
    if curl -fsSL --max-time 15 -o "$TMP_PEM" "$VLOUD_LICENSE_SERVER_URL_DEFAULT/v1/license-pubkey" \
       && grep -q 'BEGIN PUBLIC KEY' "$TMP_PEM"; then
      install -m 0644 -o root -g root "$TMP_PEM" /etc/vloud/license-v1.pub.pem
      ok "license-v1.pub.pem refreshed"
    fi
    rm -f "$TMP_PEM"
  fi

  # 2026-05-14: --repair heals known-bad ownership state. The
  # /var/lib/vloud parent was created as root:root by pre-2026-05-14
  # bootstraps and silently broke engine writes. Heal here so the
  # operator's "fix this host" knob actually fixes it.
  if id -u vloud >/dev/null 2>&1 && [[ -d /var/lib/vloud ]]; then
    cur_owner=$(stat -c '%U:%G' /var/lib/vloud 2>/dev/null)
    if [[ "$cur_owner" != "vloud:vloud" ]]; then
      say "healing /var/lib/vloud ownership: $cur_owner → vloud:vloud"
      chown vloud:vloud /var/lib/vloud 2>/dev/null
      chmod 0751 /var/lib/vloud 2>/dev/null
      [[ -d /var/lib/vloud/staging ]] && \
        chown -R vloud:vloud /var/lib/vloud/staging 2>/dev/null
      ok "/var/lib/vloud ownership healed"
    fi
  fi

  systemctl daemon-reload
  do_stop_services
  do_start_services_with_healthcheck
  do_post_upgrade_health_summary || true
  # 2026-05-14: --repair is the operator's "fix this host" knob —
  # verify all runtime deps before declaring success.
  do_verify_dependencies || warn "post-repair verify-dependencies reported failures (operator action recommended)"
  ok "REPAIR done"
}

# ── --upgrade ─────────────────────────────────────────────────
# 0.6.0b — env-file reconciliation used by BOTH fresh-install and
# upgrade flows. Pre-0.6.0b the env-write happened only inside the
# fresh-install linear phases, so `--upgrade` skipped it entirely
# and engines ended up on 0.6.0a binary with stale (or missing)
# VLOUD_LICENSE_SERVER_URL. That's what bit 188.245.113.223's
# first 0.6.0a upgrade.
#
# Idempotent. Always writes the file atomically via mktemp + install.
# Refuses to continue (returns non-zero) if a required var couldn't
# be written — fail-closed semantics so the caller surfaces a clear
# error instead of silently entering a broken state.
do_reconcile_env_file() {
  local force_url="${1:-1}"   # default: ALWAYS rewrite the cloud URL
  local TMP
  TMP=$(mktemp /etc/vloud.env.XXXXXX)

  # Resolve identity inputs. machine_id derived the same way fresh-install does.
  local raw_mid
  if [[ -f /etc/machine-id && -s /etc/machine-id ]]; then
    raw_mid=$(cat /etc/machine-id)
  else
    raw_mid=$(head -c 32 /dev/urandom | sha256sum | cut -d' ' -f1)
  fi
  local mid; mid=$(printf 'vloud:%s' "$raw_mid" | sha256sum | cut -d' ' -f1 | head -c 32)

  if [[ -f "$ENV_FILE" ]]; then
    cp "$ENV_FILE" "$TMP"
  fi

  # Local upsert: add the line only if the key is entirely absent.
  _upsert_env() {
    local k="$1" v="$2"
    if grep -q "^$k=" "$TMP"; then return; fi
    printf '%s=%s\n' "$k" "$v" >> "$TMP"
  }

  # Local rewrite: replace the value of key (or insert if missing).
  _set_env() {
    local k="$1" v="$2"
    if grep -q "^$k=" "$TMP"; then
      # Use a delimiter the URL can't contain so sed doesn't choke on '/'.
      sed -i -E "s|^$k=.*|$k=$v|" "$TMP"
    else
      printf '%s=%s\n' "$k" "$v" >> "$TMP"
    fi
  }

  # Stable bootstrap fields — upsert only (preserve existing values).
  _upsert_env PORT                  "$VLOUD_PORT"
  _upsert_env NODE_ENV              production
  _upsert_env VLOUD_BIND_HOST       "$VLOUD_BIND_HOST"
  _upsert_env VLOUD_JWT_SECRET      "$(openssl rand -hex 64)"
  _upsert_env VLOUD_SUDO_FALLBACK   0
  _upsert_env VLOUD_MACHINE_ID      "$mid"
  _upsert_env VLOUD_INSTALL_ID      "$(head -c 16 /dev/urandom | xxd -p)"
  _upsert_env VLOUD_TRIAL_EXPIRES_AT \
              "$(date -u -d "+${VLOUD_TRIAL_DAYS} days" +%Y-%m-%dT%H:%M:%SZ)"
  # Cloud email relay — enables OTP verification during onboarding.
  # Customer engines relay OTP emails through license.vloud.app instead
  # of requiring local SMTP or Resend API key configuration.
  _upsert_env VLOUD_EMAIL_RELAY_ENABLED 1
  _upsert_env VLOUD_EMAIL_RELAY_URL     "$VLOUD_LICENSE_SERVER_URL_DEFAULT"
  if [[ "${COEXIST:-0}" -eq 1 ]]; then
    _upsert_env VLOUD_COEXIST 1
    _upsert_env VLOUD_SAFE_COEXIST 1
    _upsert_env VLOUD_CUTOVER_MODE 0
  fi

  # Cloud URL — ALWAYS rewrite to the current default unless the
  # operator has set a non-default value AND force_url=0.
  #
  # The 0.6.0a default lives in VLOUD_LICENSE_SERVER_URL_DEFAULT
  # (https://license.vloud.app). Earlier defaults
  # (localhost / adminpanel.vloud.app) were broken, so we rewrite
  # them. Custom operator URLs (anything NOT matching the broken
  # set AND NOT matching the current default) are preserved.
  if [[ "$force_url" -eq 1 ]]; then
    _set_env VLOUD_LICENSE_SERVER_URL "$VLOUD_LICENSE_SERVER_URL_DEFAULT"
  else
    local existing
    existing=$(grep -E '^VLOUD_LICENSE_SERVER_URL=' "$TMP" | head -1 | cut -d= -f2- || true)
    if [[ -z "$existing" ]] \
       || echo "$existing" | grep -E '^https?://(127\.0\.0\.1|localhost|adminpanel\.vloud\.app)' >/dev/null; then
      _set_env VLOUD_LICENSE_SERVER_URL "$VLOUD_LICENSE_SERVER_URL_DEFAULT"
    fi
  fi

  # Fail-closed verification: every required var must be present + non-empty.
  local required=(PORT NODE_ENV VLOUD_MACHINE_ID VLOUD_INSTALL_ID
                  VLOUD_LICENSE_SERVER_URL VLOUD_JWT_SECRET)
  local missing=()
  for k in "${required[@]}"; do
    local v
    v=$(grep -E "^$k=" "$TMP" | head -1 | cut -d= -f2- || true)
    if [[ -z "$v" ]]; then missing+=("$k"); fi
  done
  if [[ ${#missing[@]} -gt 0 ]]; then
    rm -f "$TMP"
    fail "env-file reconcile FAILED — missing required vars: ${missing[*]}"
    return 1
  fi

  install -m 0640 -o root -g vloud "$TMP" "$ENV_FILE"
  rm -f "$TMP"
  ok "$ENV_FILE reconciled (VLOUD_LICENSE_SERVER_URL=$VLOUD_LICENSE_SERVER_URL_DEFAULT)"
  return 0
}

# 0.6.0b — post-upgrade health summary. Probes the engine's
# /api/license/state to confirm sync_status is non-disabled and
# the cloud URL is reachable. Non-zero exit signals broken state
# to the caller; do_upgrade_flow surfaces it as a warn (not a
# rollback trigger — the binary may still be perfectly serviceable
# for non-cloud workflows).
# Final dependency-and-readiness gate. Runs at the END of every
# install flow (fresh, upgrade, repair). Asserts every prerequisite
# the engine relies on is actually present + healthy. Fails LOUD
# with a clear diagnostic per missing/broken item — the install is
# considered complete only when this passes.
#
# 2026-05-14: added after the fresh-host validation found that
# bootstrap was silently skipping Redis. This gate would have caught
# that pre-release. Treat any new runtime dep as a required addition
# here.
do_verify_dependencies() {
  say "verifying runtime dependencies"
  emit_progress verify-deps started ""
  local failures=0
  local warnings=0

  # Binaries on PATH.
  local need_bin=(node npm curl openssl sqlite3 nginx redis-cli sudo systemctl)
  for b in "${need_bin[@]}"; do
    if ! command -v "$b" >/dev/null 2>&1; then
      fail "missing binary: $b"
      failures=$((failures+1))
    fi
  done

  # Node ABI sanity (we already enforced 20.x earlier; double-check).
  if command -v node >/dev/null; then
    local nmv
    nmv=$(node -e 'process.stdout.write(String(process.versions.modules))' 2>/dev/null)
    if [[ "$nmv" != "115" ]]; then
      fail "Node ABI is NMV=$nmv, expected 115 (Node 20.x)"
      failures=$((failures+1))
    fi
  fi

  # Required services. nginx is intentionally NOT hard-required here:
  # per the 0.7.0+ architecture VLoud binds 127.0.0.1:${VLOUD_PORT} and
  # does NOT claim port 80 — nginx is a gateway the operator wires up
  # later (panel domain). So nginx being inactive (e.g. another web
  # server such as Caddy/Apache already owns :80) must NOT fail a fresh
  # install when the engine itself is healthy. It's checked warn-only
  # after this loop.
  local need_service=(redis-server vloud vloud-job-worker vloud-scheduler)
  for s in "${need_service[@]}"; do
    if [[ "$s" == redis-server ]] && [[ "${VLOUD_BOOTSTRAP_NO_REDIS:-0}" == "1" ]]; then
      continue
    fi
    if [[ "$s" == vloud-job-worker || "$s" == vloud-scheduler ]] && [[ "${VLOUD_BOOTSTRAP_NO_REDIS:-0}" == "1" ]]; then
      continue
    fi
    # redis service name varies across Ubuntu generations.
    if [[ "$s" == redis-server ]]; then
      if ! systemctl is-active --quiet redis-server.service 2>/dev/null \
         && ! systemctl is-active --quiet redis.service 2>/dev/null; then
        fail "redis-server.service / redis.service is not active"
        failures=$((failures+1))
      fi
      continue
    fi
    if ! systemctl is-active --quiet "$s.service" 2>/dev/null; then
      fail "$s.service is not active"
      failures=$((failures+1))
    fi
  done

  # nginx — warn-only. The engine is fully functional on :${VLOUD_PORT}
  # without it; nginx only matters once a panel/hosting domain is
  # configured. If it isn't active, surface WHY (commonly another web
  # server already holds port 80) with an actionable next step, but do
  # not mark the install incomplete.
  if command -v nginx >/dev/null 2>&1 && ! systemctl is-active --quiet nginx.service 2>/dev/null; then
    local _p80=""
    if command -v ss >/dev/null 2>&1; then
      _p80=$(ss -ltnHp 'sport = :80' 2>/dev/null | grep -oE 'users:\(\("[^"]+"' | sed -E 's/.*"([^"]+)".*/\1/' | head -1)
    fi
    if [[ -n "$_p80" && "$_p80" != nginx ]]; then
      warn "nginx is not active — port 80 is already held by '$_p80'. VLoud runs on http://<server-ip>:${VLOUD_PORT} regardless; to serve it on 80/443 either point '$_p80' at 127.0.0.1:${VLOUD_PORT}, or stop it and 'systemctl enable --now nginx'."
    else
      warn "nginx is not active — VLoud is reachable at http://<server-ip>:${VLOUD_PORT}. Start nginx ('systemctl enable --now nginx') once you configure a panel domain."
    fi
    warnings=$((warnings+1))
  fi

  # Engine HTTP readiness — bypassed when explicitly skipped (e.g.
  # repair flow on a half-broken host that doesn't pass yet).
  if [[ "${VLOUD_VERIFY_SKIP_HTTP:-0}" != "1" ]]; then
    local rc
    rc=$(curl -s -o /dev/null -w '%{http_code}' --max-time 5 "http://127.0.0.1:$VLOUD_PORT/api/health" 2>/dev/null || echo 000)
    if [[ "$rc" != "200" ]]; then
      fail "engine /api/health returned $rc (expected 200)"
      failures=$((failures+1))
    fi
  fi

  # Redis client ping.
  if [[ "${VLOUD_BOOTSTRAP_NO_REDIS:-0}" != "1" ]]; then
    if ! redis-cli ping 2>/dev/null | grep '^PONG$' >/dev/null; then
      fail "redis-cli ping did not return PONG"
      failures=$((failures+1))
    fi
  fi

  # Required files.
  # SEA binary or legacy dist — at least one must exist.
  local engine_ok=0
  [[ -f "$VLOUD_INSTALL_DIR/vloud-engine" ]] && engine_ok=1
  [[ -f "$VLOUD_INSTALL_DIR/packages/server/dist/index.js" ]] && engine_ok=1
  [[ "$engine_ok" -eq 0 ]] && { fail "missing engine entrypoint (vloud-engine or dist/index.js)"; failures=$((failures+1)); }
  local need_file=(/etc/vloud.env /etc/vloud/license-v1.pub.pem $VLOUD_INSTALL_DIR/VERSION
                   $VLOUD_INSTALL_DIR/scripts/bootstrap.sh)
  for f in "${need_file[@]}"; do
    if [[ ! -e "$f" ]]; then
      fail "missing file: $f"
      failures=$((failures+1))
    fi
  done

  # Required listening ports. The 2544 / 6379 / 80 trio is the engine
  # contract. ss is in iproute2 (in coreutils dependency chain).
  local need_port=("$VLOUD_PORT" 80)
  if [[ "${VLOUD_BOOTSTRAP_NO_REDIS:-0}" != "1" ]]; then
    need_port+=(6379)
  fi
  for p in "${need_port[@]}"; do
    if ! ss -lnt 2>/dev/null | awk -v port=":$p$" '$4 ~ port { found=1 } END { exit !found }'; then
      fail "no process listening on port $p"
      failures=$((failures+1))
    fi
  done

  # File ownership — REPAIR, then verify, then fail if still wrong.
  #
  # The DB must be vloud:vloud, and so must its parent directory: the
  # database runs in WAL mode, so SQLite has to open vloud.db-shm even
  # to READ. Wrong ownership means every query behind /api/setup/state
  # dies with SQLITE_READONLY_DIRECTORY and the first-run wizard shows
  # a bare HTTP 500.
  #
  # This used to be a warning. An upgrade that left the tree root-owned
  # therefore printed "✓ Vloud is up." over an engine that could not
  # read its own database, and the operator was left to run chown by
  # hand. The installer's contract is one command → working system, so
  # fix it here and only fail if the fix did not take.
  if [[ -f "$VLOUD_INSTALL_DIR/packages/server/vloud.db" ]]; then
    local want owner dir_owner
    want=$(engine_service_user)
    owner=$(stat -c '%U:%G' "$VLOUD_INSTALL_DIR/packages/server/vloud.db" 2>/dev/null)
    dir_owner=$(stat -c '%U:%G' "$VLOUD_INSTALL_DIR/packages/server" 2>/dev/null)
    if [[ "$owner" != "$want:$want" || "$dir_owner" != "$want:$want" ]]; then
      warn "DB ownership is $owner (dir $dir_owner) but the engine runs as $want — repairing"
      chown -R "$want:$want" "$VLOUD_INSTALL_DIR/packages" 2>/dev/null || true
      [[ -d "$VLOUD_INSTALL_DIR/node_modules" ]] &&
        chown -R "$want:$want" "$VLOUD_INSTALL_DIR/node_modules" 2>/dev/null || true
      owner=$(stat -c '%U:%G' "$VLOUD_INSTALL_DIR/packages/server/vloud.db" 2>/dev/null)
      dir_owner=$(stat -c '%U:%G' "$VLOUD_INSTALL_DIR/packages/server" 2>/dev/null)
      if [[ "$owner" != "$want:$want" || "$dir_owner" != "$want:$want" ]]; then
        fail "DB ownership is $owner (dir $dir_owner) and could not be repaired — the engine cannot write its database"
        failures=$((failures+1))
      else
        ok "repaired DB ownership → $want:$want"
        # The engine is already running against the old permissions.
        systemctl restart vloud >/dev/null 2>&1 || true
        sleep 3
      fi
    fi
  fi

  # Prove it: a read through the engine's own API, which is what the
  # wizard calls. Ownership can look right and still fail (SELinux,
  # a stale mount), and this check is the difference between "the
  # install reported success" and "the install works".
  if [[ -f "$VLOUD_INSTALL_DIR/packages/server/vloud.db" ]]; then
    local setup_code
    for _ in 1 2 3 4 5; do
      setup_code=$(curl -s -o /dev/null -w '%{http_code}' --max-time 5 \
        "http://127.0.0.1:${VLOUD_PORT}/api/setup/state" 2>/dev/null || echo 000)
      [[ "$setup_code" == "200" ]] && break
      sleep 2
    done
    if [[ "$setup_code" != "200" ]]; then
      fail "engine cannot serve /api/setup/state (HTTP $setup_code) — the first-run wizard will not load"
      fail "check: journalctl -u vloud -n 50 | grep -i sqlite"
      failures=$((failures+1))
    else
      ok "first-run wizard reachable (/api/setup/state → 200)"
    fi
  fi

  # /var/lib/vloud writable by vloud user.
  if ! sudo -u vloud test -w /var/lib/vloud 2>/dev/null; then
    warn "/var/lib/vloud is not writable by user 'vloud' (state-write paths will fail)"
    warnings=$((warnings+1))
  fi

  if [[ $failures -gt 0 ]]; then
    emit_progress verify-deps failed "$failures dependency check(s) failed"
    fail "$failures dependency check(s) failed — install is INCOMPLETE"
    fail "see 'systemctl status vloud vloud-job-worker vloud-scheduler redis-server' + 'journalctl -u vloud' for diagnostics"
    return 1
  fi
  if [[ $warnings -gt 0 ]]; then
    warn "$warnings non-fatal dependency warning(s) above"
  fi
  emit_progress verify-deps ok "all dependencies healthy"
  ok "all runtime dependencies present + healthy"
  return 0
}

do_post_upgrade_health_summary() {
  echo
  say "post-upgrade health summary"
  printf "  %-28s %s\n" "engine /api/health" \
    "$(curl -s --max-time 4 "http://127.0.0.1:${VLOUD_PORT}/api/health" 2>/dev/null | head -c 200)"

  local ver
  ver=$(curl -s --max-time 4 "http://127.0.0.1:${VLOUD_PORT}/api/system/version" 2>/dev/null \
        | sed -n 's/.*"display":"\([^"]*\)".*/\1/p')
  printf "  %-28s %s\n" "engine version" "${ver:-?}"

  local env_url
  env_url=$(grep -E '^VLOUD_LICENSE_SERVER_URL=' "$ENV_FILE" 2>/dev/null | cut -d= -f2-)
  printf "  %-28s %s\n" "/etc/vloud.env URL" "${env_url:-(missing)}"

  local pubkey_state="missing"
  [[ -s /etc/vloud/license-v1.pub.pem ]] && pubkey_state="present ($(wc -c </etc/vloud/license-v1.pub.pem) bytes)"
  printf "  %-28s %s\n" "/etc/vloud/license-v1.pub.pem" "$pubkey_state"

  local cloud_reach="unreachable"
  if [[ -n "$env_url" ]]; then
    if curl -fsS --max-time 5 "$env_url/v1/license-pubkey" >/dev/null 2>&1; then
      cloud_reach="ok"
    fi
  fi
  printf "  %-28s %s\n" "cloud /v1/license-pubkey" "$cloud_reach"

  echo
  if [[ "$env_url" == "$VLOUD_LICENSE_SERVER_URL_DEFAULT" \
        && "$pubkey_state" != "missing" \
        && "$cloud_reach" == "ok" ]]; then
    ok "cloud sync prerequisites OK — engine should register within ~60s"
    return 0
  fi
  warn "cloud sync prerequisites NOT FULLY MET. See /var/log/vloud/install.log."
  return 1
}

do_upgrade_flow() {
  say "UPGRADE: atomic swap with rollback"
  emit_progress upgrade-start started "target=${VLOUD_TARGET_VERSION:-latest}"
  if [[ ! -d "$VLOUD_INSTALL_DIR/packages/server/dist" ]] && [[ ! -f "$VLOUD_INSTALL_DIR/vloud-engine" ]]; then
    die "no existing install at $VLOUD_INSTALL_DIR — use --fresh instead" 3
  fi

  # Runtime packages FIRST, before touching the install.
  #
  # An upgrade must be able to install dependencies a newer release
  # introduced. Without this the upgrade path only ever swapped code:
  # MariaDB and Docker were added to the package lists and every
  # already-installed host still had neither, silently, because nothing
  # on this path ever ran apt. Idempotent — already-present packages
  # are a no-op — and it runs before the swap so a package failure
  # aborts while the existing install is still intact.
  ensure_system_packages

  # Capture current version for progress events. Read from the
  # installed tree's VERSION file (canonical) before we touch anything.
  if [[ -f "$VLOUD_INSTALL_DIR/VERSION" ]]; then
    VLOUD_VERSION_FROM=$(cat "$VLOUD_INSTALL_DIR/VERSION" 2>/dev/null | tr -d '[:space:]')
  fi
  [[ -n "$VLOUD_TARGET_VERSION" ]] && VLOUD_VERSION_TO="$VLOUD_TARGET_VERSION"

  local ts; ts=$(date -u +%Y%m%dT%H%M%SZ)
  local staging=/opt/vloud-staging-$ts
  local rollback=/opt/vloud.pre-upgrade-$ts

  # 1. Backup state BEFORE touching anything.
  # do_backup_state emits its own backup started/ok progress, naming the
  # archive; a second bare pair here would report the stage twice.
  do_backup_state "pre-upgrade" >/dev/null

  # 2. Download + verify the new release into staging (a fresh dir
  #    next to /opt/vloud, NOT inside it, so we can atomic-mv on success
  #    and rm-rf on failure).
  # Refuse to start an upgrade without room for the staging tree.
  # tar failing halfway through leaves a half-extracted install and a
  # dead engine; a clear refusal beforehand is far kinder.
  local _free_mb
  _free_mb=$(df -Pm "$(dirname "$VLOUD_INSTALL_DIR")" 2>/dev/null | awk 'NR==2{print $4}')
  if [[ -n "$_free_mb" && "$_free_mb" -lt 1500 ]]; then
    die "only ${_free_mb} MB free on $(dirname "$VLOUD_INSTALL_DIR") — an upgrade needs ~1.5 GB for the staging tree.
       Free space and retry. Old rollback trees are the usual culprit:
         sudo du -sh ${VLOUD_INSTALL_DIR}.pre-upgrade-* 2>/dev/null
         sudo rm -rf \$(ls -d ${VLOUD_INSTALL_DIR}.pre-upgrade-* | sort | head -n -2)" 5
  fi

  say "downloading new release to $staging"
  emit_progress downloading started "$VLOUD_RELEASE_URL"
  install -d -m 0755 "$staging"
  local TMPTAR TMPSHA TMPSIG
  TMPTAR=$(mktemp /tmp/vloud-upgrade-XXXXXX.tar.gz)
  TMPSHA="$TMPTAR.sha256"
  TMPSIG="$TMPTAR.sig"
  if ! curl -fsSL --max-time 600 -o "$TMPTAR" "$VLOUD_RELEASE_URL"; then
    rm -f "$TMPTAR" "$TMPSHA" "$TMPSIG"; rm -rf "$staging"
    die "release download failed: $VLOUD_RELEASE_URL" 4
  fi
  if [[ "${VLOUD_INSECURE_SKIP_VERIFY:-0}" != "1" ]]; then
    if ! curl -fsSL --max-time 60 -o "$TMPSHA" "$VLOUD_RELEASE_URL.sha256"; then
      rm -f "$TMPTAR" "$TMPSHA" "$TMPSIG"; rm -rf "$staging"
      die "release sha256 download failed: $VLOUD_RELEASE_URL.sha256" 4
    fi
    if ! curl -fsSL --max-time 60 -o "$TMPSIG" "$VLOUD_RELEASE_URL.sig"; then
      rm -f "$TMPTAR" "$TMPSHA" "$TMPSIG"; rm -rf "$staging"
      die "release sig download failed: $VLOUD_RELEASE_URL.sig" 4
    fi
    # Rewrite sha256 line to match the on-disk mktemp filename.
    local SHA_HEX
    SHA_HEX=$(awk '{print $1}' "$TMPSHA")
    printf '%s  %s\n' "$SHA_HEX" "$(basename "$TMPTAR")" > "$TMPSHA"
    local EXPECTED_ARTIFACT
    EXPECTED_ARTIFACT="$(basename "${VLOUD_RELEASE_URL%%\?*}")"
    emit_progress verifying started "verify_release_chain"
    verify_release_chain "$TMPTAR" "$TMPSHA" "$TMPSIG" "$EXPECTED_ARTIFACT"
    emit_progress verifying ok "manifest signature + sha256 verified"
  else
    if [[ "${VLOUD_TESTING:-0}" != "1" ]]; then
      rm -f "$TMPTAR"; rm -rf "$staging"
      die "VLOUD_INSECURE_SKIP_VERIFY=1 requires VLOUD_TESTING=1 (refusing on production host)" 4
    fi
    warn "VERIFICATION SKIPPED — VLOUD_INSECURE_SKIP_VERIFY=1 (test mode)"
    emit_progress verifying warn "skipped (test mode)"
  fi
  tar -xzf "$TMPTAR" -C "$staging" --strip-components=1
  rm -f "$TMPTAR" "$TMPSHA" "$TMPSIG"
  ok "release extracted to $staging"
  emit_progress downloading ok "extracted to $staging"

  # 2a. Per-file integrity (PR1) — defence-in-depth on top of the
  #     signed manifest's tarball-SHA. Catches single-file corruption
  #     in the staging dir before we swap it into /opt/vloud.
  do_verify_sha256sums "$staging"

  # 2b. ABI preflight — refuse to swap if the release's prebuilt
  #     native modules can't load against the host's Node. The
  #     0.5.1-beta upgrade against 188.245.113.223 (2026-05-13)
  #     shipped better-sqlite3 built for NODE_MODULE_VERSION 127 to
  #     a host with NODE_MODULE_VERSION 115, the new engine couldn't
  #     boot, and the resulting systemd start-limit-hit cascade
  #     poisoned the rollback. Fail-closed BEFORE the swap.
  emit_progress abi-preflight started ""
  do_abi_preflight_or_rebuild "$staging" || {
    rm -rf "$staging"
    emit_progress abi-preflight failed "rebuild failed; swap refused"
    die "ABI preflight failed — refusing to swap; existing install untouched" 8
  }
  emit_progress abi-preflight ok ""

  # 3. Stop services BEFORE the swap so they don't half-read the new tree.
  emit_progress stop-services started ""
  do_stop_services
  emit_progress stop-services ok ""

  # 4. Atomic swap: rename current → rollback, staging → current.
  say "swapping $VLOUD_INSTALL_DIR ↔ $rollback (atomic)"
  emit_progress swap started "$VLOUD_INSTALL_DIR ↔ $rollback"
  mv "$VLOUD_INSTALL_DIR" "$rollback"
  mv "$staging" "$VLOUD_INSTALL_DIR"
  emit_progress swap ok ""

  # 5. Re-link state: copy the live DB from rollback into the new tree.
  if [[ -f "$rollback/packages/server/vloud.db" ]]; then
    cp -a "$rollback/packages/server/vloud.db"     "$VLOUD_INSTALL_DIR/packages/server/vloud.db"
    [[ -f "$rollback/packages/server/vloud.db-wal" ]] && cp -a "$rollback/packages/server/vloud.db-wal" "$VLOUD_INSTALL_DIR/packages/server/vloud.db-wal"
    [[ -f "$rollback/packages/server/vloud.db-shm" ]] && cp -a "$rollback/packages/server/vloud.db-shm" "$VLOUD_INSTALL_DIR/packages/server/vloud.db-shm"
    ok "preserved DB from rollback tree"
  fi
  # Note: /etc/vloud.env was NEVER touched — it's outside $VLOUD_INSTALL_DIR.
  # Same for /var/lib/vloud (engine state, geoip, snapshots).

  # 5a. CRITICAL: normalize ownership of the swapped-in tree. The
  # staging dir was created by `tar -xzf` + `npm install` running as
  # root, so after `mv staging -> /opt/vloud` everything is root:root.
  # The cp -a above preserves vloud:vloud on the DB FILE, but the
  # PARENT DIR (/opt/vloud/packages/server/) stays root:root 755.
  # SQLite in WAL mode needs to CREATE vloud.db-wal / vloud.db-shm
  # in that parent dir at boot — without group-write or vloud
  # ownership, it fails with SQLITE_READONLY_DIRECTORY and background
  # workers (monitor collector, guardian, migration worker, ssl
  # worker) crash on every write. This bit 0.5.1 → 0.5.3 upgrades.
  #
  # Ownership MUST match the unit's User=, and is derived from it —
  # never hardcoded on either side.
  #
  # The engine runs as root but WITHOUT CAP_DAC_OVERRIDE:
  # packaging/systemd/vloud.service pins
  # `CapabilityBoundingSet=CAP_NET_BIND_SERVICE CAP_SETUID CAP_SETGID`.
  # Being uid 0 therefore grants no permission bypass at all — ordinary
  # DAC checks apply, so a root-run engine cannot write a vloud-owned
  # 0644 database. It fails with SQLITE_READONLY ("attempt to write a
  # readonly database"), which surfaces as HTTP 500 on the public
  # /api/setup/state and a first-run wizard that will not load.
  #
  # Hardcoding either half is how this broke before: the units moved to
  # User=root while do_verify_dependencies still expected vloud:vloud.
  # Read the effective user from the unit so the two cannot disagree.
  local engine_user
  engine_user=$(engine_service_user)
  chown -R "$engine_user:$engine_user" "$VLOUD_INSTALL_DIR/packages" 2>/dev/null \
    || warn "chown of packages/ partial"
  if [[ -d "$VLOUD_INSTALL_DIR/node_modules" ]]; then
    chown -R "$engine_user:$engine_user" "$VLOUD_INSTALL_DIR/node_modules" 2>/dev/null || true
  fi
  chmod 0755 "$VLOUD_INSTALL_DIR/packages/server" 2>/dev/null || true
  ok "normalized ownership: $VLOUD_INSTALL_DIR/packages → $engine_user:$engine_user"

  if id -u vloud >/dev/null 2>&1; then

    # 2026-05-14: heal /var/lib/vloud parent ownership. Pre-this-date
    # bootstraps created the dir as root:root (bug at line ~1778),
    # which silently broke engine state-writes — caught by the
    # do_verify_dependencies gate during fresh-host validation. Always
    # reconcile on upgrade so hosts on the buggy old bootstrap heal
    # automatically on the next upgrade tick. Subdirs (backups,
    # crontabs, acme) keep their own ownership.
    if [[ -d /var/lib/vloud ]]; then
      cur_owner=$(stat -c '%U:%G' /var/lib/vloud 2>/dev/null)
      if [[ "$cur_owner" != "vloud:vloud" ]]; then
        say "healing /var/lib/vloud ownership: $cur_owner → vloud:vloud"
        chown vloud:vloud /var/lib/vloud 2>/dev/null
        chmod 0751 /var/lib/vloud 2>/dev/null
        # Re-stamp the engine-owned subtrees (staging/) just in case
        # they were also clobbered. backups/, crontabs/, acme/ keep
        # their canonical owners (root:root + root:www-data).
        for d in staging; do
          [[ -d /var/lib/vloud/$d ]] && \
            chown -R vloud:vloud "/var/lib/vloud/$d" 2>/dev/null
        done
        ok "/var/lib/vloud ownership healed"
      fi
    fi
  else
    warn "vloud user not present — skipping ownership normalization (engine will fail to write)"
  fi

  # 5a-new. 0.6.0b — env-file reconciliation. The upgrade flow used
  # to skip this entirely (only the fresh-install linear phases
  # touched /etc/vloud.env), so 0.6.0 → 0.6.0a upgrades that
  # depended on a new env var landed broken. Fail-closed: if the
  # env file isn't sane post-reconcile, abort the upgrade (the
  # rollback tree is still on disk, so this is a recoverable point).
  if ! do_reconcile_env_file 1; then
    fail "env-file reconcile failed — refusing to start services"
    do_stop_services
    rm -rf "$VLOUD_INSTALL_DIR"
    mv "$rollback" "$VLOUD_INSTALL_DIR"
    do_start_services_with_healthcheck || die "rollback also failed after env-reconcile abort" 7
    die "upgrade aborted, previous version restored (env-reconcile failed)" 7
  fi

  # 5a-new. 0.6.0a — refresh the cloud license-signing pubkey.
  if [[ -n "${VLOUD_LICENSE_SERVER_URL_DEFAULT:-}" ]]; then
    say "refreshing /etc/vloud/license-v1.pub.pem from $VLOUD_LICENSE_SERVER_URL_DEFAULT"
    install -d -m 0755 /etc/vloud
    TMP_PEM=$(mktemp /tmp/vloud-license-pubkey.XXXXXX.pem)
    if curl -fsSL --max-time 15 -o "$TMP_PEM" "$VLOUD_LICENSE_SERVER_URL_DEFAULT/v1/license-pubkey" \
       && grep -q 'BEGIN PUBLIC KEY' "$TMP_PEM"; then
      install -m 0644 -o root -g root "$TMP_PEM" /etc/vloud/license-v1.pub.pem
      ok "license-v1.pub.pem refreshed"
    else
      warn "could not refresh license-v1.pub.pem; cloud-pull will retry"
    fi
    rm -f "$TMP_PEM"
  fi

  # 5b. VLoud nginx reverse proxy.
  #
  # Architecture (0.7.0+): VLoud binds to 127.0.0.1:2544 ONLY.
  # It does NOT claim port 80 or default_server. nginx acts as a
  # gateway — the operator configures a panel domain during onboarding,
  # and VLoud generates a domain-specific vhost (no wildcard catch-all).
  #
  # For initial onboarding (before a domain is configured), the operator
  # accesses VLoud directly at http://SERVER-IP:2544/onboarding.
  #
  # Migration: if a vloud-default vhost exists from a pre-0.7.0 install,
  # remove it so VLoud no longer claims default_server. The operator
  # can re-enable it via Settings > Panel Domain if needed.
  if [[ -f /etc/nginx/sites-enabled/vloud-default ]]; then
    say "migrating: removing vloud-default (VLoud no longer claims port 80)"
    rm -f /etc/nginx/sites-enabled/vloud-default
    if nginx -t 2>&1 | grep -E 'test is successful|syntax is ok' >/dev/null; then
      systemctl reload nginx 2>/dev/null || true
      ok "vloud-default removed — access VLoud at http://<server-ip>:${VLOUD_PORT}"
    fi
  fi

  # 5c. Sync systemd units from the new release into /etc/systemd/system/.
  # Without this, the engine keeps running under the OLD unit config
  # (e.g. User=vloud) even after an upgrade ships User=root.
  if [[ -d "$VLOUD_INSTALL_DIR/packaging/systemd" ]]; then
    for unit in vloud.service vloud-job-worker.service vloud-scheduler.service; do
      if [[ -f "$VLOUD_INSTALL_DIR/packaging/systemd/$unit" ]]; then
        cp "$VLOUD_INSTALL_DIR/packaging/systemd/$unit" "/etc/systemd/system/$unit"
      fi
    done
    systemctl daemon-reload 2>/dev/null || true
    ok "systemd units synced from release"
  fi

  # 5d. Webmail provisioner helpers + sudoers grants from the NEW release.
  #
  # The fresh-install linear phases install the webmail helpers into
  # /usr/local/sbin and render /etc/sudoers.d/vloud from the release's
  # scripts/sudoers.d/vloud-locked — but --upgrade dispatches here and
  # exits before those phases, so without this an upgrade ships the new
  # engine yet leaves webmail unable to run and any grants added since the
  # installed version missing. ensure_system_packages() (called at the top
  # of this flow) already set SUDO_BIN / VISUDO_BIN, so bootstrap-sudoers.sh
  # validates with the correct visudo on sudo-rs hosts. Both are idempotent
  # and must land BEFORE the engine restarts below so it sees the grants.
  for _wm in install install-sogo uninstall; do
    _wm_src="$VLOUD_INSTALL_DIR/scripts/vloud-webmail-${_wm}.sh"
    [[ -f "$_wm_src" ]] && install -m 0755 -o root -g root "$_wm_src" "/usr/local/sbin/vloud-webmail-${_wm}"
  done
  # External-repo CI/CD host state — same rationale as the helpers above:
  # --upgrade dispatches here and exits BEFORE the fresh linear phases, so a
  # host upgrading THROUGH the release that introduced this feature would never
  # get the pipeline execution account or the artifact/release roots at all.
  # The engine then refuses every pipeline step (no vloud-ci) and every deploy
  # (no release root) on a box that looks perfectly healthy.
  #
  # Every line below is idempotent, so re-running an upgrade is a no-op.
  if ! getent group vloud-ci >/dev/null 2>&1; then
    groupadd --system vloud-ci && ok "created vloud-ci system group"
  fi
  if ! id -u vloud-ci >/dev/null 2>&1; then
    useradd --system --no-create-home --home-dir /nonexistent \
      --shell /usr/sbin/nologin --gid vloud-ci vloud-ci \
      && ok "created vloud-ci pipeline execution user"
    passwd -l vloud-ci >/dev/null 2>&1 || true
  fi
  # 0711 on the workspace root: the CI account must be able to TRAVERSE into
  # its own run directory but must not be able to LIST the root and discover
  # other runs.
  install -d -m 0711 -o root -g root /var/lib/vloud/pipeline-workspaces
  # Engine-owned. `vloud-ci` never writes here — the engine copies artifacts out
  # of the workspace after the step completes.
  install -d -m 0750 -o root -g root /var/lib/vloud/artifacts
  install -d -m 0751 -o root -g root /var/lib/vloud/apps

  # Caddy-fronted domain provisioner — same rationale as the webmail helpers:
  # --upgrade skips the fresh linear phases, so install it here too (before the
  # engine restart below) so a Caddy box gains domain serving on upgrade.
  _dom_up_src="$VLOUD_INSTALL_DIR/scripts/vloud-domain-apply.sh"
  [[ -f "$_dom_up_src" ]] && install -m 0755 -o root -g root "$_dom_up_src" "/usr/local/sbin/vloud-domain-apply"
  # Native VCS helpers live outside the install dir and are NOT refreshed by
  # the atomic swap, so --upgrade must reinstall them here — the same rationale
  # as the webmail and domain helpers above. Without this an upgraded host keeps
  # whatever binary the previous release installed, and a host upgrading THROUGH
  # the release that introduced vloud-import-contain would never get it at all,
  # leaving Azure DevOps import permanently gated by I.G10.
  install -d -m 0755 -o root -g root /usr/local/libexec/vloud
  _vcs_up_src="$VLOUD_INSTALL_DIR/packages/server"
  if [[ -f "$_vcs_up_src/native/vloud-vcs-lock-supervisor/vloud-vcs-lock-supervisor" ]]; then
    install -m 0755 -o root -g root \
      "$_vcs_up_src/native/vloud-vcs-lock-supervisor/vloud-vcs-lock-supervisor" \
      /usr/local/libexec/vloud-vcs-lock-supervisor
  elif [[ -f "$_vcs_up_src/native/vloud-vcs-lock-supervisor/vloud-vcs-lock-supervisor.c" ]] && command -v cc >/dev/null 2>&1; then
    make -C "$_vcs_up_src/native/vloud-vcs-lock-supervisor" >/dev/null 2>&1 \
      && install -m 0755 -o root -g root \
        "$_vcs_up_src/native/vloud-vcs-lock-supervisor/vloud-vcs-lock-supervisor" \
        /usr/local/libexec/vloud-vcs-lock-supervisor
  fi
  if [[ -f "$_vcs_up_src/native/vloud-import-contain/vloud-import-contain" ]]; then
    install -m 0755 -o root -g root \
      "$_vcs_up_src/native/vloud-import-contain/vloud-import-contain" \
      /usr/local/libexec/vloud/vloud-import-contain
    ok "refreshed vloud-import-contain"
  elif [[ -f "$_vcs_up_src/native/vloud-import-contain/vloud-import-contain.c" ]] && command -v cc >/dev/null 2>&1; then
    if make -C "$_vcs_up_src/native/vloud-import-contain" >/dev/null 2>&1; then
      install -m 0755 -o root -g root \
        "$_vcs_up_src/native/vloud-import-contain/vloud-import-contain" \
        /usr/local/libexec/vloud/vloud-import-contain
      ok "compiled and refreshed vloud-import-contain"
    else
      warn "could not build vloud-import-contain — Azure DevOps import will stay closed (I.G10)"
    fi
  fi
  if [[ -f "$VLOUD_INSTALL_DIR/scripts/bootstrap-sudoers.sh" ]]; then
    if bash "$VLOUD_INSTALL_DIR/scripts/bootstrap-sudoers.sh" >/dev/null 2>&1; then
      ok "sudoers grants + webmail helpers refreshed from new release"
    else
      warn "sudoers re-render reported an issue — webmail/new grants may be missing"
    fi
  fi

  # Build tooling on PATH — same rationale as the helpers above: corepack
  # shims live in /usr/local/bin, outside the atomic swap, so a host upgrading
  # INTO the release that added this would otherwise never get them and its
  # pipeline builds would keep dying on `pnpm: not found`.
  ensure_corepack_shims

  # 6. Start + health-check. On failure, roll back.
  emit_progress healthcheck started "starting services + curl /api/health"
  if do_start_services_with_healthcheck; then
    # Prune old rollback trees. Each is a full copy of the install
  # (~207 MB with node_modules) and nothing used to remove them, so
  # every upgrade leaked one: twelve upgrades filled a 25 GB disk to
  # 100%, tar failed mid-extract and the engine died with SQLITE_FULL.
  # Keep the two most recent so a rollback is still possible.
  local _keep=2 _old
  mapfile -t _old < <(ls -d "${VLOUD_INSTALL_DIR}".pre-upgrade-* 2>/dev/null | sort | head -n "-${_keep}")
  if (( ${#_old[@]} > 0 )); then
    rm -rf "${_old[@]}" 2>/dev/null || true
    ok "pruned ${#_old[@]} old rollback tree(s), kept the newest $_keep"
  fi
  # Staging dirs are scratch — a failed run can leave one behind.
  rm -rf "${VLOUD_INSTALL_DIR}"-staging-* 2>/dev/null || true

  ok "upgrade succeeded — new version is live"
    emit_progress healthcheck ok "new version healthy"
    # Cleanup old rollback dirs older than 7 days.
    find /opt -maxdepth 1 -type d -name 'vloud.pre-upgrade-*' -mtime +7 -exec rm -rf {} + 2>/dev/null || true
    say "version after upgrade:"
    curl -s --max-time 4 "http://127.0.0.1:$VLOUD_PORT/api/system/version" 2>&1 | head -c 400; echo
    # 0.6.0b — surface cloud-sync prerequisites. Warn-only (non-fatal)
    # because a broken cloud-sync state is recoverable via --repair
    # and shouldn't trigger an automatic rollback.
    do_post_upgrade_health_summary || true
    # 2026-05-14: dependency gate at the END of the upgrade. Failure
    # here means the new version is up + serving but something in
    # the runtime environment regressed (redis disabled, worker
    # crash, port not listening). Surface loudly; the operator can
    # rollback via 'bootstrap.sh --rollback' if needed.
    do_verify_dependencies || warn "post-upgrade verify-dependencies reported failures"
  else
    fail "new version failed health-check — rolling back"
    emit_progress healthcheck failed "engine did not become healthy in 30s"
    do_dump_failure_journal
    emit_progress auto-rollback started "restoring $rollback"
    do_stop_services
    rm -rf "$VLOUD_INSTALL_DIR"
    mv "$rollback" "$VLOUD_INSTALL_DIR"
    if ! do_start_services_with_healthcheck; then
      do_dump_failure_journal
      emit_progress auto-rollback failed "rollback restart failed"
      die "rollback also failed — manual intervention required. See $LOG_FILE and journalctl -u vloud. To recover: systemctl reset-failed vloud.service && systemctl start vloud.service" 5
    fi
    emit_progress auto-rollback ok "previous version restored"
    die "upgrade aborted, previous version restored" 6
  fi

  ok "UPGRADE done"
}

# ─── --rollback: operator-triggered restore from .pre-upgrade-<ts>/ ───
#
# Two forms:
#   bootstrap.sh --rollback               → restore newest slot
#   bootstrap.sh --rollback --to <v|path> → restore matching slot
#
# Slot semantics: `/opt/vloud.pre-upgrade-<ts>/` directories are
# created by do_upgrade_flow's atomic swap. Each holds the FULL
# previous /opt/vloud tree. Operator picks one to restore; we
# reverse the swap.
#
# Safety:
#   - DB is preserved by copying the CURRENT live DB INTO the slot
#     before the swap (otherwise rolling back also rolls the DB
#     back, which is rarely what the operator wants).
#   - The pre-rollback tree is archived as
#     /opt/vloud.pre-rollback-<ts>/ so we can roll forward again
#     if the operator changes their mind.
#   - Healthcheck after restart; on failure, undo the rollback
#     (re-swap to the pre-rollback tree).
do_rollback_flow() {
  say "ROLLBACK: restore from .pre-upgrade-<ts>/ slot"
  emit_progress rollback-start started "to=${VLOUD_ROLLBACK_TO:-newest}"

  if [[ ! -d "$VLOUD_INSTALL_DIR" ]]; then
    die "no live install at $VLOUD_INSTALL_DIR — nothing to roll back from" 3
  fi

  # 1. Resolve the target slot.
  local slot=""
  if [[ -n "$VLOUD_ROLLBACK_TO" ]]; then
    # --to may be an explicit path OR a version string.
    if [[ -d "$VLOUD_ROLLBACK_TO" ]]; then
      slot="$VLOUD_ROLLBACK_TO"
    elif [[ -d "/opt/$VLOUD_ROLLBACK_TO" ]]; then
      slot="/opt/$VLOUD_ROLLBACK_TO"
    else
      # Treat as version label; find a slot whose VERSION matches.
      while IFS= read -r d; do
        if [[ -f "$d/VERSION" ]] && [[ "$(tr -d '[:space:]' <"$d/VERSION")" == "$VLOUD_ROLLBACK_TO" ]]; then
          slot="$d"; break
        fi
      done < <(find /opt -maxdepth 1 -type d -name 'vloud.pre-upgrade-*' 2>/dev/null | sort -r)
    fi
    if [[ -z "$slot" ]]; then
      die "no rollback slot matches --to '$VLOUD_ROLLBACK_TO' (looked for explicit path and matching VERSION)" 3
    fi
  else
    # No --to: pick the newest .pre-upgrade-* slot.
    slot=$(find /opt -maxdepth 1 -type d -name 'vloud.pre-upgrade-*' 2>/dev/null | sort -r | head -n1)
    if [[ -z "$slot" ]]; then
      die "no .pre-upgrade-* slots available — nothing to roll back to" 3
    fi
  fi
  local slot_version="(unknown)"
  if [[ -f "$slot/VERSION" ]]; then
    slot_version=$(tr -d '[:space:]' <"$slot/VERSION")
  fi
  say "rollback target: $slot (version: $slot_version)"
  emit_progress rollback-resolve ok "slot=$slot version=$slot_version"

  # Capture from/to for progress payloads.
  if [[ -f "$VLOUD_INSTALL_DIR/VERSION" ]]; then
    VLOUD_VERSION_FROM=$(tr -d '[:space:]' <"$VLOUD_INSTALL_DIR/VERSION")
  fi
  VLOUD_VERSION_TO="$slot_version"

  # 2. Pre-rollback backup.
  do_backup_state "pre-rollback" >/dev/null

  # 3. Optional: verify slot integrity (best-effort — older slots
  #    may predate SHA256SUMS).
  do_verify_sha256sums "$slot" || true

  # 4. Stop services.
  emit_progress stop-services started ""
  do_stop_services
  emit_progress stop-services ok ""

  # 5. Preserve the LIVE DB into the slot before swap. Without this,
  #    rolling back also rolls the DB back; that's a foot-gun.
  if [[ -f "$VLOUD_INSTALL_DIR/packages/server/vloud.db" ]]; then
    cp -a "$VLOUD_INSTALL_DIR/packages/server/vloud.db" \
          "$slot/packages/server/vloud.db" 2>/dev/null || \
      warn "could not preserve live DB into slot (rollback will use slot's older DB)"
    for ext in -wal -shm; do
      [[ -f "$VLOUD_INSTALL_DIR/packages/server/vloud.db$ext" ]] && \
        cp -a "$VLOUD_INSTALL_DIR/packages/server/vloud.db$ext" \
              "$slot/packages/server/vloud.db$ext" 2>/dev/null
    done
    ok "preserved live DB into rollback slot"
  fi

  # 6. Reverse swap. Archive the current tree as pre-rollback-<ts>/
  #    so we can roll forward again if needed.
  local ts; ts=$(date -u +%Y%m%dT%H%M%SZ)
  local archive=/opt/vloud.pre-rollback-$ts
  say "swapping $VLOUD_INSTALL_DIR → $archive ; $slot → $VLOUD_INSTALL_DIR"
  emit_progress swap started "rollback swap"
  mv "$VLOUD_INSTALL_DIR" "$archive"
  mv "$slot"              "$VLOUD_INSTALL_DIR"
  emit_progress swap ok ""

  # 7. Normalize ownership on the restored tree — to the unit's User=,
  #    for the CAP_DAC_OVERRIDE reason documented at the upgrade path.
  local rb_user
  rb_user=$(engine_service_user)
  chown -R "$rb_user:$rb_user" "$VLOUD_INSTALL_DIR/packages" 2>/dev/null || true
  if [[ -d "$VLOUD_INSTALL_DIR/node_modules" ]]; then
    chown -R "$rb_user:$rb_user" "$VLOUD_INSTALL_DIR/node_modules" 2>/dev/null || true
  fi
  chmod 0775 "$VLOUD_INSTALL_DIR/packages/server" 2>/dev/null || true

  # 8. Start + healthcheck. On failure, undo (restore the archive).
  emit_progress healthcheck started ""
  if do_start_services_with_healthcheck; then
    ok "rollback succeeded — engine on version $slot_version"
    emit_progress healthcheck ok "engine healthy on $slot_version"
  else
    fail "post-rollback healthcheck failed — undoing rollback"
    emit_progress healthcheck failed "engine did not become healthy"
    do_dump_failure_journal
    do_stop_services
    rm -rf "$VLOUD_INSTALL_DIR"
    mv "$archive" "$VLOUD_INSTALL_DIR"
    if ! do_start_services_with_healthcheck; then
      emit_progress rollback-undo failed "could not restore pre-rollback tree"
      die "rollback failed AND undo failed — manual intervention required" 5
    fi
    emit_progress rollback-undo ok "pre-rollback tree restored"
    die "rollback aborted — engine restored to pre-rollback state" 6
  fi

  ok "ROLLBACK done"
}

# ── engine purge (shared by --force-clean-install and --uninstall) ──
# Removes the Vloud engine, its systemd units, install dir, state, DB,
# env, sudoers/logrotate, and the engine-generated nginx vhosts. Does
# NOT touch the OS, SSH access, or the installed daemon-stack packages
# (mail/db/etc.) — that fuller wipe is scripts/host-nuke.sh.
_vloud_purge_engine() {
  # Best-effort teardown: NEVER abort on a single failed step. The
  # installer runs under `set -euo pipefail`; a non-zero from any cleanup
  # command (a service already stopped, a file already gone, a
  # `daemon-reload` racing a concurrent daemon-stack install) would
  # otherwise exit the script mid-purge and strand /opt/vloud + the units
  # while the engine keeps running. Suspend errexit for the duration.
  set +e
  do_stop_services
  systemctl disable vloud.service vloud-job-worker.service vloud-scheduler.service 2>/dev/null || true
  rm -f /etc/systemd/system/vloud.service \
        /etc/systemd/system/vloud-job-worker.service \
        /etc/systemd/system/vloud-scheduler.service \
        /etc/systemd/system/vloud.slice
  rm -rf /etc/systemd/system/vloud.service.d \
         /etc/systemd/system/vloud-job-worker.service.d \
         /etc/systemd/system/vloud-scheduler.service.d
  systemctl daemon-reload
  rm -rf "$VLOUD_INSTALL_DIR" "$ENV_FILE" /var/lib/vloud /var/log/vloud
  rm -f  /etc/sudoers.d/vloud /etc/logrotate.d/vloud
  # Engine-generated nginx vhosts only (preserve commercial + customer vhosts).
  while IFS= read -r f; do rm -f "$f"; done < <(
    ls /etc/nginx/sites-enabled/vloud-*.conf 2>/dev/null |
    grep -vE '/vloud-(app-|admin|commercial-|marketing|portal|license)' || true
  )
  # 0.5.3 default vhost (extensionless symlink, not in the .conf glob above).
  rm -f /etc/nginx/sites-enabled/vloud-default /etc/nginx/sites-available/vloud-default
  nginx -t 2>/dev/null && systemctl reload nginx 2>/dev/null
  set -e
}

# ── --force-clean-install (purge then fresh) ──────────────────
do_force_clean_purge() {
  say "FORCE-CLEAN: purging existing install before fresh"
  _vloud_purge_engine
  ok "purge complete — proceeding with fresh install"
}

# ── --uninstall (purge only — no reinstall) ───────────────────
# The curl-pipe uninstall one-liner:
#   curl -fsSL https://install.vloud.app | sudo bash -s -- --uninstall
# Requires a typed "UNINSTALL" confirmation (read from /dev/tty so it
# works through the curl pipe); add --yes for unattended.
do_uninstall_flow() {
  say "UNINSTALL: removing the Vloud engine + state (no reinstall)"
  say "  removes: engine, systemd units, $VLOUD_INSTALL_DIR, /var/lib/vloud (DB),"
  say "           $ENV_FILE, sudoers, logrotate, engine nginx vhosts"
  say "  keeps:   the OS, your SSH access, and installed daemon packages"
  if [[ "${VLOUD_ASSUME_YES:-0}" != "1" ]]; then
    # Read the confirmation from the controlling terminal (not stdin —
    # stdin is the curl pipe). Probe that /dev/tty can actually be
    # OPENED, not just that the device node has read perms.
    if { true < /dev/tty; } 2>/dev/null; then
      printf "Type UNINSTALL to proceed (or anything else to abort): " > /dev/tty
      read -r _ans < /dev/tty || _ans=""
      if [[ "$_ans" != "UNINSTALL" ]]; then
        warn "aborted — nothing was removed"
        exit 1
      fi
    else
      die "refusing --uninstall without confirmation (no terminal for the prompt). Re-run unattended: curl -fsSL https://install.vloud.app | sudo bash -s -- --uninstall --yes" 2
    fi
  fi
  _vloud_purge_engine
  ok "Vloud engine + state removed"
  say "reinstall any time:  curl -fsSL https://install.vloud.app | sudo bash"
  say "full host wipe (daemon stack + tenant users too): scripts/host-nuke.sh"
}

# ── System packages ───────────────────────────────────────────────────
#
# Wrapped in a function so the UPGRADE path can run it too.
#
# This block used to live inline in the linear fresh-install flow, and
# `upgrade)` dispatches to do_upgrade_flow and exits long before
# reaching it. The consequence: an upgrade NEVER installed packages, so
# any runtime dependency added by a later release was silently absent
# on every already-installed host. MariaDB and Docker were added to the
# lists and upgraded machines still had neither — no error, because
# nothing ever tried.
#
# apt/dnf install of an already-present package is a no-op, so calling
# this on every path is safe and idempotent.
ensure_system_packages() {
  say "installing system packages"
  pkg_update
  ensure_rhel_repos   # no-op on Debian; enables EPEL + Remi + CRB on RHEL

  if [[ "$OS_FAMILY" == debian ]]; then
    # Core: things the engine + bootstrap itself reach for via shell-out.
    # `git` and `util-linux` are Native VCS prerequisites (plan gate 0.G2).
    # git was previously absent from every package array on BOTH branches —
    # the only `git` token in this whole script was a comment — and util-linux
    # (which provides flock(1), used by the cross-process TEST harness and the
    # Phase 0 gates, never by a production lock path) was likewise undeclared.
    # Declaring them beats relying on them being Essential/@core in practice.
    PKG_CORE=(curl ca-certificates gnupg lsb-release jq tar gzip rsync
              openssh-client unzip whois dnsutils sqlite3 openssl
              git util-linux)

    # Runtime services the engine speaks to as a CLIENT (not the
    # daemon-stack, which is a separate concern — those are mail/DNS/FTP
    # servers the engine *manages* for tenants, installed later by
    # bootstrap-daemon-stack.sh).
    PKG_RUNTIME=(
      # Redis: BullMQ queues (workers/job-worker.ts) + scheduler pubsub
      # (workers/scheduler.ts). Both default to redis://127.0.0.1:6379.
      # vloud.service systemd unit lists `After=redis.service`. Without
      # redis the engine boots but workers crash-loop.
      redis-server
      # nginx: reverse proxy in front of the engine on :80 → :2544. The
      # release UI ships its SPA assets via nginx, not the engine itself.
      nginx
      # MariaDB: the tenant database server. Distinct from the engine's
      # own SQLite. The engine's health probe looks for /usr/sbin/mariadbd
      # (falling back to mysqld) and then mariadb.service — see
      # services/health-probes.ts — and the MySQL plugin provisions
      # per-account databases through it. Neither this nor Docker used to
      # be installed at all, so a shared-hosting install came up with no
      # database server and the panel reported MySQL stopped forever.
      mariadb-server
      # Docker: the runtime behind marketplace deploys and the
      # `docker` app-discovery source (services/app-discovery.ts shells
      # out to the docker CLI). docker.io is Ubuntu/Debian's own build —
      # no third-party repo, which keeps the install self-contained.
      docker.io
      # Compose v2, as the `docker compose` SUBCOMMAND. docker.io does not
      # pull this in, and the v1 `docker-compose` binary is a DIFFERENT
      # command: a project whose scripts call `docker compose` fails with
      # "docker: unknown command", which surfaces as exit 125 from whatever
      # invoked it rather than as a missing dependency.
      docker-compose-plugin
      # PostgreSQL: offered as a first-class module in the wizard
      # ("Per-tenant Postgres with role isolation") but never installed,
      # so selecting it did nothing and the capability reported "no
      # install mapping". MongoDB is deliberately NOT here — Ubuntu does
      # not package it at all, and the wizard already describes it as a
      # managed Docker container.
      postgresql
      # The Postgres CLIENT tools, separately from the server. pg_dump,
      # pg_restore and psql are what an import or migration actually runs,
      # and they are needed even when the server is remote — restoring a
      # managed cloud database into Vloud otherwise fails at the first
      # command. pg_dump also REFUSES to dump a server newer than itself,
      # so the client version is a real constraint, not a convenience.
      postgresql-client
    )

    # Build toolchain — kept on every host because better-sqlite3 and
    # node-pty fall back to source-compile when prebuilt-install can't
    # match the host's NMV. The toolchain is also needed by `npm install`
    # itself for postinstall scripts. ~150 MB.
    PKG_BUILD=(build-essential python3 python3-dev)

    # SSL automation. Vloud's hosting-accounts flow uses certbot for
    # Let's Encrypt issuance + renewal.
    PKG_SSL=(certbot)

    # PHP runtime for the hosting-accounts feature. Even on hosts where
    # the operator never deploys a PHP app, the PHP install costs ~80 MB
    # and saves an apt-install round-trip at first deploy time. Treat as
    # a known cost — there's a long-term TODO to defer this until the
    # first PHP app is provisioned.
    # PHP version is DETECTED, not pinned. Vloud targets 8.3 — what
    # Ubuntu 22.04/24.04 and Remi on RHEL 9 provide — but newer Ubuntu
    # releases carry no php8.3 packages at all. 26.04 (resolute) ships
    # 8.5 only, and the ondrej PPA publishes no resolute suite, so a
    # hardcoded 8.3 turned every install on that release into a wall of
    # "Unable to locate package php8.3-*" and left the host half-built.
    #
    # Order: prefer 8.3 when the host has it (the tested configuration),
    # otherwise take the newest phpX.Y-fpm the repos actually offer.
    PHP_VERSION="$(detect_php_version)" || die "No phpX.Y-fpm package is available from this host's apt repositories.
         Vloud needs PHP-FPM for the hosting-accounts feature.
         Check that the universe component is enabled:
           sudo add-apt-repository universe && sudo apt-get update" 9
    if [[ "$PHP_VERSION" != "8.3" ]]; then
      say "php8.3 unavailable on $PRETTY_NAME — using php$PHP_VERSION"
    fi
    PKG_HOSTING_PHP=(
      "php${PHP_VERSION}-fpm"       "php${PHP_VERSION}-cli"
      "php${PHP_VERSION}-mbstring"  "php${PHP_VERSION}-xml"
      "php${PHP_VERSION}-mysql"     "php${PHP_VERSION}-curl"
      "php${PHP_VERSION}-gd"        "php${PHP_VERSION}-zip"
      "php${PHP_VERSION}-intl"      "php${PHP_VERSION}-bcmath"
    )
  else
    # RHEL-family equivalents. Package-name deltas vs Debian:
    #   gnupg→gnupg2, openssh-client→openssh-clients, dnsutils→bind-utils,
    #   sqlite3→sqlite, lsb-release→redhat-lsb-core. semanage lives in
    #   policycoreutils-python-utils (needed by configure_selinux).
    # git + util-linux — same Native VCS prerequisite as the Debian branch
    # (plan gate 0.G2). Same package names on RHEL.
    PKG_CORE=(curl ca-certificates gnupg2 jq tar gzip rsync
              openssh-clients unzip whois bind-utils sqlite openssl
              policycoreutils-python-utils
              git util-linux)

    # redis (not redis-server); nginx same name. The redis-enable loop
    # below already probes both redis.service and redis-server.service.
    # redis (not redis-server) on RHEL; mariadb-server is the same name.
    # Docker is NOT in RHEL 9 base — podman-docker provides a `docker`
    # shim that satisfies app-discovery's CLI shell-out without pulling
    # in a third-party repo during bootstrap.
    # podman-compose gives RHEL the `docker compose` equivalent alongside
    # the podman-docker shim; postgresql supplies psql/pg_dump/pg_restore,
    # which this branch was missing for the same reason Debian was.
    PKG_RUNTIME=(redis nginx mariadb-server podman-docker podman-compose
                 postgresql-server postgresql)

    # Toolchain — gcc/gcc-c++/make replace build-essential; python3-devel
    # replaces python3-dev. Needed for the same native-module fallback.
    PKG_BUILD=(gcc gcc-c++ make python3 python3-devel)

    # certbot + the nginx plugin come from EPEL.
    PKG_SSL=(certbot python3-certbot-nginx)

    # PHP 8.3 from the Remi module enabled by ensure_rhel_repos.
    # php-mysqlnd replaces php-mysql; php-process provides posix/shmop.
    PKG_HOSTING_PHP=(php-fpm php-cli php-mbstring php-xml php-mysqlnd
                     php-gd php-zip php-intl php-bcmath php-process)
    # Unversioned on RHEL: one php-fpm service serves whatever the Remi
    # module selected, so the adapter's singleService path ignores this.
    # Still recorded for the engine's pool-path construction.
    PHP_VERSION=8.3
  fi
  export PHP_VERSION

  # Record what is running so we can prove we did not break it. Taken as late
  # as possible before the first package touches the system.
  snapshot_running_services

  # Never stand a second database server next to a working one.
  # Plain word-splitting rather than mapfile: package names never contain
  # whitespace, and this works on bash 3.2 too. The function's warnings go to
  # stderr, so they reach the operator (and the tee'd install log) instead of
  # being swallowed into this capture.
  PKG_RUNTIME=($(drop_db_if_present ${PKG_RUNTIME[@]+"${PKG_RUNTIME[@]}"}))

  install_packages "${PKG_CORE[@]}" ${PKG_RUNTIME[@]+"${PKG_RUNTIME[@]}"} "${PKG_BUILD[@]}" "${PKG_SSL[@]}" "${PKG_HOSTING_PHP[@]}"
  report_package_decisions
  verify_services_survived
  ok "system packages ready ($OS_FAMILY)"

  # ── Which sudo? ───────────────────────────────────────────────────────
  # Ubuntu 26.04 makes `sudo-rs` the default sudo. It rejects wildcards in
  # command arguments, which is the form all 32 of Vloud's daemon sudoers
  # rules use, so every daemon fragment fails `visudo -c` and the mail /
  # DNS / FTP / firewall / quota stack refuses to install.
  #
  # Classic sudo is still packaged, installed alongside under `.ws` names.
  # Detect and prefer it. Duplicated rather than sourced from
  # scripts/_os-family.sh because this script is curl-piped and has no
  # sibling files — keep the two in step.
  SUDO_BIN=/usr/bin/sudo
  if command -v sudo >/dev/null 2>&1 && sudo --version 2>&1 | grep -i 'sudo-rs' >/dev/null; then
    if [[ "$OS_FAMILY" == debian ]] && [[ ! -x /usr/bin/sudo.ws ]]; then
      install_packages sudo >/dev/null 2>&1 || true
    fi
    if [[ -x /usr/bin/sudo.ws && -x /usr/sbin/visudo.ws ]]; then
      SUDO_BIN=/usr/bin/sudo.ws
      export VISUDO_BIN=/usr/sbin/visudo.ws
      ok "sudo-rs detected — using classic sudo ($SUDO_BIN) for wildcard sudoers rules"
    else
      warn "sudo-rs is the only sudo available; daemon sudoers rules using wildcards will be rejected"
    fi
  fi
  export SUDO_BIN

  # Drop Vloud sudoers fragments that no longer parse.
  #
  # sudo evaluates /etc/sudoers.d as a unit: one unparseable file makes
  # every rule in the directory suspect, so a stale fragment breaks sudo
  # for rules that are themselves valid. Fragments written before the
  # sudo-rs switch use wildcards in command arguments and are already
  # dead weight. Each daemon script rewrites its own immediately after,
  # so removing them is not destructive.
  #
  # Mirrors prune_invalid_sudoers_fragments() in scripts/_os-family.sh;
  # duplicated because this script is curl-piped and has no siblings.
  # Checked with the SYSTEM visudo, not VISUDO_BIN: those differ on
  # 26.04, and it is the system parser that decides whether the operator's
  # own `sudo` works.
  local _sysvisudo; _sysvisudo="$(command -v visudo 2>/dev/null || true)"
  if [[ -d /etc/sudoers.d && -n "$_sysvisudo" ]]; then
    local _f
    for _f in /etc/sudoers.d/vloud-*; do
      [[ -f "$_f" ]] || continue
      if ! "$_sysvisudo" -c -f "$_f" >/dev/null 2>&1; then
        rm -f "$_f"
        warn "removed stale sudoers fragment $(basename "$_f") — it no longer parses"
      fi
    done
  fi

  # Redis MUST be enabled + running before we start the engine — the
  # vloud.service unit has Wants=/After=redis.service so systemd will
  # wait for it, but if redis is masked/disabled the wait turns into a
  # silent fail. Fail-fast here with an actionable diagnostic.
  say "enabling + starting redis"
  emit_progress install-redis started ""
  # Ubuntu 24.04 ships redis as redis-server.service. Older Ubuntus
  # used redis.service — try both for forward+back compat.
  REDIS_UNIT=
  for u in redis-server.service redis.service; do
    # `systemctl cat`, not `list-unit-files`: the listing misses a unit
    # registered moments earlier in the same apt run, so a FRESH install
    # that had just installed redis reported "neither redis-server.service
    # nor redis.service registered with systemd" and died — while the unit
    # was present. Same trap as enable_optional_unit above.
    if systemctl cat "$u" >/dev/null 2>&1; then
      REDIS_UNIT="$u"; break
    fi
  done
  if [[ -z "$REDIS_UNIT" ]]; then
    emit_progress install-redis failed "redis unit not registered after apt install"
    die "redis-server package installed but neither redis-server.service nor redis.service registered with systemd. Inspect: dpkg -l redis-server; systemctl list-unit-files | grep -i redis" 8
  fi
  systemctl enable --now "$REDIS_UNIT" >/dev/null 2>&1
  # Wait up to 15s for redis-cli ping → PONG. ping is the canonical
  # readiness probe; redis is in-process at that point so it's <100ms
  # usually but we budget for slow VMs.
  ready=0
  for i in 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15; do
    if redis-cli ping 2>/dev/null | grep '^PONG$' >/dev/null; then ready=1; break; fi
    sleep 1
  done
  if [[ "$ready" -ne 1 ]]; then
    emit_progress install-redis failed "redis-cli ping never returned PONG"
    fail "redis-cli ping did not return PONG within 15s — check 'journalctl -u $REDIS_UNIT'"
    die "redis readiness check failed" 8
  fi
  ok "redis ready ($REDIS_UNIT, port 6379)"

  # ── MariaDB + Docker ──────────────────────────────────────────────────
  # Installed above but not started by default on every distro. Enable
  # them so the panel reflects a working stack rather than a half-built
  # one. Non-fatal: unlike redis the engine boots fine without either —
  # tenant databases and container deploys are the features that degrade.
  enable_optional_unit() { # enable_optional_unit <label> <unit>...
    local label="$1"; shift
    local unit
    for unit in "$@"; do
      # Judge by outcome, not by `list-unit-files`. That listing misses
      # a unit registered moments earlier in the same apt run and misses
      # alias/generated units: PostgreSQL installed, started, and was
      # still reported "no service unit found (package missing?)" while
      # postgresql.service was demonstrably active.
      systemctl cat "$unit" >/dev/null 2>&1 || continue
      systemctl enable --now "$unit" >/dev/null 2>&1
      if [[ "$(systemctl is-active "$unit" 2>/dev/null)" == "active" ]]; then
        ok "$label ready ($unit)"
        return 0
      fi
      warn "$label: $unit did not become active — 'journalctl -u $unit' has the detail"
      return 1
    done
    warn "$label: no service unit found (package missing?) — feature unavailable"
    return 1
  }

  say "enabling + starting MariaDB"
  # Debian names it mariadb.service; some builds still ship mysql.service.
  enable_optional_unit "MariaDB" mariadb.service mysql.service mysqld.service || true

  say "enabling + starting Docker"
  enable_optional_unit "Docker" docker.service podman.socket || true

  say "enabling + starting PostgreSQL"
  enable_optional_unit "PostgreSQL" postgresql.service postgresql@18-main.service || true
}

# ─── Dispatch ───
# This is where we either run a mode-specific flow and exit, or fall
# through to the linear fresh-install phases below.
case "$MODE" in
  doctor)
    do_doctor_flow
    exit 0
    ;;
  repair)
    emit_progress dispatch started "repair flow"
    do_repair_flow
    rc=$?
    if [[ $rc -eq 0 ]]; then emit_progress_terminal ok "repair complete"; else emit_progress_terminal failed "repair exit=$rc"; fi
    exit $rc
    ;;
  upgrade)
    emit_progress dispatch started "upgrade flow target=${VLOUD_TARGET_VERSION:-latest}"
    do_upgrade_flow
    rc=$?
    if [[ $rc -eq 0 ]]; then emit_progress_terminal ok "upgrade complete"; else emit_progress_terminal failed "upgrade exit=$rc"; fi
    exit $rc
    ;;
  rollback)
    emit_progress dispatch started "rollback flow target=${VLOUD_ROLLBACK_TO:-newest}"
    do_rollback_flow
    rc=$?
    if [[ $rc -eq 0 ]]; then emit_progress_terminal ok "rollback complete"; else emit_progress_terminal failed "rollback exit=$rc"; fi
    exit $rc
    ;;
  uninstall)
    do_uninstall_flow
    exit 0
    ;;
  force-clean)
    do_force_clean_purge
    # Fall through to fresh install below.
    ;;
  fresh)
    if [[ -d "$VLOUD_INSTALL_DIR/packages/server/dist" ]]; then
      die "$VLOUD_INSTALL_DIR exists — refusing destructive --fresh. Use --upgrade or --force-clean-install." 7
    fi
    ;;
  *)
    die "unknown install mode: $MODE" 2
    ;;
esac
# (fresh / force-clean continue with phases 1-6 below)

# dpkg prints amd64/arm64 (Debian); uname -m prints x86_64/aarch64
# (RHEL has no dpkg). Normalise both to the Debian spelling.
ARCH=$(dpkg --print-architecture 2>/dev/null || uname -m)
case "$ARCH" in
  amd64|x86_64)  ARCH=amd64; ok "arch: $ARCH" ;;
  arm64|aarch64) ARCH=arm64; ok "arch: $ARCH" ;;
  *) die "unsupported architecture: $ARCH (need amd64/x86_64 or arm64/aarch64)" ;;
esac

# Disk check — engine + app data + DB needs ~3GB minimum.
DISK_FREE_GB=$(df -BG --output=avail / | tail -1 | tr -d 'G ')
if [[ "$DISK_FREE_GB" -lt 3 ]]; then
  die "/, only ${DISK_FREE_GB}GB free; need at least 3GB"
fi
ok "disk: ${DISK_FREE_GB}GB free"

# RAM check — 1GB hard floor, 2GB recommended.
MEM_MB=$(awk '/MemTotal/ {print int($2/1024)}' /proc/meminfo)
if [[ "$MEM_MB" -lt 1024 ]]; then
  warn "ram: ${MEM_MB}MB (1GB hard floor — engine may be unstable)"
elif [[ "$MEM_MB" -lt 2048 ]]; then
  warn "ram: ${MEM_MB}MB (2GB recommended)"
else
  ok "ram: ${MEM_MB}MB"
fi

# Outbound network — apt and the release tarball both need it.  We only
# warn on failure here; the apt step will surface the real error.
if curl -fsS --max-time 5 -o /dev/null -w '' https://deb.debian.org/ 2>/dev/null \
   || curl -fsS --max-time 5 -o /dev/null -w '' https://archive.ubuntu.com/ 2>/dev/null; then
  ok "outbound HTTPS reachable"
else
  warn "outbound HTTPS appears unreachable (apt + release download may fail)"
fi

# ── Node 20 must be OBTAINABLE, and we settle that BEFORE touching the host ──
#
# This check used to live after the package phase, and that ordering did real
# damage on 2026-08-27: the installer restarted the host's whole service stack,
# swapped its database server, and only THEN discovered it could not get Node
# 20 and aborted. The operator was left with every side effect of an install
# and no Vloud — strictly worse than if the script had never run.
#
# A prerequisite that can fail belongs before the first irreversible act, not
# after the last one. Nothing above this point has modified the system beyond
# apt metadata.
if [[ "$OS_FAMILY" == debian ]]; then
  # gpg and curl are needed to even ASK the question. Additive, tiny, and
  # install_packages leaves them alone if they are already present.
  install_packages ca-certificates curl gnupg >/dev/null 2>&1 || true
fi
if node20_obtainable; then
  ok "Node 20 available$( node --version >/dev/null 2>&1 && echo " (host has $(node --version))" )"
else
  die "Node 20.x is required and this host cannot obtain it: $(node --version 2>/dev/null || echo 'no node installed') is present and no 20.x candidate is available from apt.
  Nothing has been changed on this host — the check runs before any package or service is touched.
  Fix the Node source and re-run, e.g.:
    sudo apt-get install -y ca-certificates curl gnupg
    curl -fsSL https://deb.nodesource.com/setup_20.x | sudo -E bash -
    sudo apt-get install -y nodejs" 7
fi

# Detect cPanel — set later in /etc/vloud.env so the engine knows to coexist.
if [[ -d /usr/local/cpanel || -d /var/cpanel ]]; then
  COEXIST=1
  SAFE_COEXIST=1
  cat <<'EOSAFE'

  ┌──────────────────────────────────────────────────────────────────────┐
  │  WHM/cPanel DETECTED — entering WHM SAFE COEXIST MODE (record-only)     │
  ├──────────────────────────────────────────────────────────────────────┤
  │  Vloud will NOT perform a normal clean-server install and will NOT     │
  │  touch your live panel. Service takeover is DISABLED until you finish  │
  │  the cutover prerequisites and enable cutover explicitly.              │
  │                                                                        │
  │  ALLOWED now:                                                          │
  │   • Phase-1 read-only audit (run as root to capture all gaps)          │
  │   • Phase-2 records-first, DB-only import into Vloud                   │
  │   • Operator UI/API on coexist ports (UI 9080/9443, API 127.0.0.1:9500)│
  │                                                                        │
  │  BLOCKED now:                                                          │
  │   • Binding 80/443/25/465/587/110/143/993/995/53/3306                  │
  │   • Writing nginx/apache/postfix/exim/dovecot/bind/powerdns configs    │
  │   • Provisioning MySQL on the shared engine                            │
  │   • Writing /home, DNS zones, CSF/firewall, or cron                    │
  │   • Restarting production services / creating OS users                 │
  │   • Shutting down WHM/cPanel or any live-service takeover              │
  └──────────────────────────────────────────────────────────────────────┘

EOSAFE
  ok "cPanel detected — WHM safe-coexist mode (engine binds port ${VLOUD_PORT} only)"
  # Refuse the normal clean-server provisioning path on a live panel unless the
  # operator explicitly opts into the record-only coexist install.
  if [[ "${VLOUD_CONFIRM_SAFE_COEXIST:-0}" != "1" ]]; then
    die "Refusing clean-server provisioning on a WHM/cPanel host. Re-run with VLOUD_CONFIRM_SAFE_COEXIST=1 to install Vloud in safe coexist (record-only) mode."
  fi
else
  COEXIST=0
  SAFE_COEXIST=0
  ok "cPanel not present"
fi

# Port checks — refuse if our port is taken by something we don't recognise.
if ss -ltn "sport = :$VLOUD_PORT" 2>/dev/null | tail -n +2 | grep . >/dev/null; then
  # Distinguish between "we are already running" (idempotent re-run) and
  # "something else is listening".
  if pgrep -f "node.*vloud.*dist/index" >/dev/null 2>&1; then
    ok "port $VLOUD_PORT in use by an existing Vloud engine — will restart"
  elif systemctl is-active --quiet vloud.service 2>/dev/null; then
    ok "port $VLOUD_PORT held by vloud.service — will restart"
  else
    die "port $VLOUD_PORT already bound by another process (set VLOUD_PORT= to override)"
  fi
fi

# ─── 2. Apt install ───
#
# Vloud's runtime dependencies, grouped by concern. Every package
# below is REQUIRED for a working engine on a fresh Ubuntu 24.04
# host — the installer contract is "single command → fully working
# system", which means we don't get to ask the operator to apt-install
# anything afterward. Each group has a comment explaining why; if you
# remove something, document the new contract first.
#
# 2026-05-14: added redis-server (gap caught in fresh-host validation
# — engine workers depend on redis but it wasn't being installed).
# See docs/operations/host-nuke-and-revalidate.md.

# Fresh-install flow runs it here, where the inline block used to be.
ensure_system_packages

emit_progress install-redis ok "$REDIS_UNIT ready"

# Node 20 LTS (NodeSource). Vloud standardizes on Node 20 — the
# release tarball ships better-sqlite3 + bcrypt prebuilt-binaries
# against NODE_MODULE_VERSION 115 (Node 20.x). A non-20 Node here
# means native modules will fail to load. We REINSTALL when the
# installed Node isn't 20.x — not just when it's missing — to repair
# hosts that were previously installed against Node 18 or Node 22.
need_node=0
if ! command -v node >/dev/null; then need_node=1
elif [[ $(node --version | sed -E 's/^v([0-9]+).*/\1/') -ne 20 ]]; then
  warn "found $(node --version) — Vloud requires Node 20.x, will reinstall"
  need_node=1
fi
if [[ "$need_node" -eq 1 ]]; then
  if [[ "$OS_FAMILY" == debian ]]; then
    configure_nodesource20
    # --allow-downgrades: the pin makes NodeSource's 20.x the candidate,
    # but apt still refuses to move a host DOWN from a distro-shipped 22
    # without being told to ("Packages were downgraded and -y was used
    # without --allow-downgrades"). Repairing such a host is precisely
    # the intent. No-op where Node is absent or already 20.
    apt-get install -y -qq --allow-downgrades nodejs
  else
    # RHEL: NodeSource ships an rpm setup script that wires the dnf repo.
    curl -fsSL https://rpm.nodesource.com/setup_20.x | bash - >/dev/null 2>&1 \
      || warn "NodeSource rpm setup script failed — will try dnf module"
    dnf install -y -q nodejs \
      || { dnf module reset -y nodejs >/dev/null 2>&1; dnf module enable -y nodejs:20 >/dev/null 2>&1; dnf install -y -q nodejs; }
  fi
fi
# Post-install assertion. Feasibility was already settled in preflight, before
# anything was touched, so reaching this with the wrong major means the install
# itself misbehaved rather than the host being unsuitable — worth saying so,
# because the two have completely different remedies.
NODE_MAJOR=$(node --version | sed -E 's/^v([0-9]+).*/\1/')
if [[ "$NODE_MAJOR" -ne 20 ]]; then
  die "Node 20.x was available in preflight but the host is running $(node --version) after the install step. A NodeSource repo may have been re-added mid-install. Check /etc/apt/sources.list.d/ for a nodesource source other than nodesource.list." 7
fi
LOCAL_NMV=$(node -e 'process.stdout.write(String(process.versions.modules))')
ok "node $(node --version) (NODE_MODULE_VERSION=$LOCAL_NMV), npm $(npm --version)"

ensure_corepack_shims

# pm2 — NOT optional. It supervises every application the engine deploys; the
# systemd units here run the ENGINE, not the customer's apps.
if ! command -v pm2 >/dev/null; then
  npm install -g --silent pm2 2>/dev/null && ok "pm2 installed" || warn "pm2 install skipped"
fi

# Give pm2 its own systemd unit, and start it BEFORE the engine can use it.
#
# The daemon is started lazily by the first `pm2` command. That command comes
# from vloud-job-worker, so without this the daemon lands in the WORKER's
# cgroup — and systemd's default KillMode=control-group then kills it, and
# every application it supervises, every time the worker restarts. An engine
# upgrade restarts the worker, so upgrading Vloud took every deployed customer
# app offline until someone started them by hand.
#
# Observed on a live host: "pm2 has been killed by signal" in pm2.log at the
# exact second of the upgrade, and
#   God Daemon (/root/.pm2) -> 0::/vloud.slice/vloud-job-worker.service
#
# Starting pm2-root.service here means the daemon already exists when the
# worker first calls pm2, so pm2 talks to it over its RPC socket instead of
# spawning one inside our cgroup. Idempotent: safe on install and upgrade.
if command -v pm2 >/dev/null; then
  if ! systemctl is-active --quiet pm2-root 2>/dev/null; then
    pm2 startup systemd -u root --hp /root >/dev/null 2>&1 || true
    if systemctl list-unit-files 2>/dev/null | grep -q '^pm2-root\.service'; then
      systemctl enable --now pm2-root >/dev/null 2>&1 \
        && ok "pm2 runs under systemd (survives engine upgrades)" \
        || warn "pm2-root.service could not be started; apps may stop on upgrade"
    else
      warn "pm2 startup did not create pm2-root.service; apps may stop on upgrade"
    fi
  fi
fi

# ─── 3. Vloud release download + verification ───
#
# Install-time verification chain (matches vloud-commercial Slice 2):
#   1. Download tarball + .sha256 + .sig.
#   2. Verify sha256 matches.
#   3. Verify Ed25519 JWS over {artifact, version, channel, sha256}.
#   4. Cross-check payload sha256 + artifact name.
#   5. Only after all checks pass: tar -xzf.
#
# Refusal modes (refuse-never-warn):
#   - sha256 mismatch         → die with hash detail
#   - signature missing       → die with URL hint
#   - signature invalid       → die
#   - artifact-name mismatch  → die (prevents repurposed-sig attack)
#
# The release public key is supplied via $VLOUD_RELEASE_PUBKEY_PEM
# (env var) or, post-build-substitution, the embedded heredoc below.
# Production builds of bootstrap.sh substitute the placeholder
# `__VLOUD_RELEASE_PUBKEY_PEM__` with the real PEM at release time.
#
# ESCAPE HATCH for testing only: VLOUD_INSECURE_SKIP_VERIFY=1 skips
# step 3.  Refuses to run on Ubuntu/Debian production hosts unless
# the operator also sets VLOUD_TESTING=1 (so accidental skips on a
# real install loudly require both).

# (EMBEDDED_RELEASE_PUBKEY_PEM, b64url_decode, verify_release_chain
# were hoisted above the mode dispatcher in the 0.5.1 multi-mode
# installer rewrite. Both fresh + upgrade flows call them.)

say "fetching Vloud release"
if [[ -d "$VLOUD_INSTALL_DIR/packages/server/dist" ]]; then
  ok "$VLOUD_INSTALL_DIR exists — keeping existing release (re-run safe)"
else
  install -d -m 0755 "$VLOUD_INSTALL_DIR"
  TMPTAR=$(mktemp /tmp/vloud-release-XXXXXX.tar.gz)
  TMPSHA="$TMPTAR.sha256"
  TMPSIG="$TMPTAR.sig"
  trap 'rm -f "$TMPTAR" "$TMPSHA" "$TMPSIG"' EXIT
  if ! curl -fsSL --max-time 600 -o "$TMPTAR" "$VLOUD_RELEASE_URL"; then
    die "release download failed: $VLOUD_RELEASE_URL"
  fi
  # Companion files.  Skip-verify mode (test only) tolerates 404s.
  if [[ "${VLOUD_INSECURE_SKIP_VERIFY:-0}" != "1" ]]; then
    if ! curl -fsSL --max-time 60 -o "$TMPSHA" "$VLOUD_RELEASE_URL.sha256"; then
      die "release sha256 download failed: $VLOUD_RELEASE_URL.sha256"
    fi
    if ! curl -fsSL --max-time 60 -o "$TMPSIG" "$VLOUD_RELEASE_URL.sig"; then
      die "release sig download failed: $VLOUD_RELEASE_URL.sig"
    fi
    # The sha256sum -c check expects the basename of the tarball
    # in the .sha256 file to match what's on disk; our tmpfile is
    # named differently, so rewrite the line to match.
    SHA_HEX=$(awk '{print $1}' "$TMPSHA")
    printf '%s  %s\n' "$SHA_HEX" "$(basename "$TMPTAR")" > "$TMPSHA"
    # The signed JWS payload names the artifact as published on the
    # release host (e.g. vloud-latest.tar.gz). Pass that name through
    # so the cross-check compares published-name vs signed-name, not
    # tmpfile-name vs signed-name.
    EXPECTED_ARTIFACT="$(basename "${VLOUD_RELEASE_URL%%\?*}")"
    verify_release_chain "$TMPTAR" "$TMPSHA" "$TMPSIG" "$EXPECTED_ARTIFACT"
  else
    if [[ "${VLOUD_TESTING:-0}" != "1" ]]; then
      die "VLOUD_INSECURE_SKIP_VERIFY=1 requires VLOUD_TESTING=1 (refusing on production host)"
    fi
    warn "VERIFICATION SKIPPED — VLOUD_INSECURE_SKIP_VERIFY=1 set (test mode)"
  fi
  tar -xzf "$TMPTAR" -C "$VLOUD_INSTALL_DIR" --strip-components=1
  rm -f "$TMPTAR" "$TMPSHA" "$TMPSIG"
  ok "release extracted to $VLOUD_INSTALL_DIR"
fi

# Sanity — engine entrypoint must exist.
# SEA binary format (0.7.0+): vloud-engine at install root
# Legacy format (0.6.x): packages/server/dist/index.js
if [[ -f "$VLOUD_INSTALL_DIR/vloud-engine" ]]; then
  VLOUD_ENGINE_FORMAT=sea
  say "detected SEA binary release"
  chmod +x "$VLOUD_INSTALL_DIR/vloud-engine"
elif [[ -f "$VLOUD_INSTALL_DIR/packages/server/dist/index.js" ]]; then
  VLOUD_ENGINE_FORMAT=legacy
  say "detected legacy dist release"
else
  die "release missing both vloud-engine and dist/index.js — corrupt download or wrong layout"
fi

if [[ "$VLOUD_ENGINE_FORMAT" == "legacy" ]]; then
# Rebuild native bindings against the installed Node ABI. The release
# tarball is built on whatever Node version the packager used; if that
# differs from the customer's Node major, better-sqlite3's .node file
# fails with "Module did not self-register" at every db query.
# `npm rebuild` re-runs the install scripts which pull the matching
# prebuilt binary (or compile from source if no prebuilt exists). Cheap
# and idempotent — costs a few seconds when the binary already matches.
say "rebuilding native bindings against local Node $(node --version)"
if ( cd "$VLOUD_INSTALL_DIR/packages/server" \
     && npm rebuild better-sqlite3 --no-audit --no-fund 2>/dev/null \
        | grep 'rebuilt dependencies' >/dev/null ) \
   || ( cd "$VLOUD_INSTALL_DIR/packages/server" \
        && npm rebuild better-sqlite3 --no-audit --no-fund >/dev/null 2>&1 ); then
  ok "native bindings ABI-matched to local Node"
else
  warn "npm rebuild reported non-zero; install build-essential + python3 if a source compile is needed"
fi
fi  # end VLOUD_ENGINE_FORMAT == legacy

# ─── 4. Configure ───
say "configuring vloud system user + env file"

# vloud user (system, no shell).
if ! id -u vloud >/dev/null 2>&1; then
  useradd --system --home-dir /var/lib/vloud --shell /usr/sbin/nologin --user-group vloud
  ok "created vloud system user"
else
  ok "vloud user exists"
fi
# /var/lib/vloud is 0751, NOT 0750: the Git root below it is root:git, and the
# `git` service account is in no group that owns this directory. Without the
# world-execute bit the account cannot TRAVERSE to its own root, and every
# `git init --bare` fails with "could not initialize the bare repository".
# 0751 grants traverse only — not read — so the directory still cannot be
# listed, and each child keeps gating access with its own mode. This is the
# same pattern /var/lib/vloud/apps (0751) already uses.
install -d -m 0751 -o vloud -g vloud /var/lib/vloud
install -d -m 0750 -o vloud -g vloud /var/log/vloud

# ---------------------------------------------------------------------------
# Native VCS `git` service account (plan D11 / D11a, gate 0.G5).
#
# All tenant-controlled Git object processing runs as this account, never as
# root — the engine itself is User=root, so without a dedicated account every
# Git child would process tenant object data as uid 0.
#
# NOTE THE DELIBERATE DIVERGENCE FROM THE vloud ACCOUNT ABOVE: this one gets
# `--shell /bin/sh`, NOT `--shell /usr/sbin/nologin`. OpenSSH executes a forced
# command THROUGH the account's login shell, as `<shell> -c "<command>"`, so an
# account whose shell is nologin cannot reliably execute the forced VCS command
# at all — nologin prints its refusal and exits non-zero regardless of the
# argument it is handed. Requiring nologin AND a forced-command helper are
# mutually exclusive. Do not "fix" this back.
#
# The shell is therefore NOT the security boundary. Interactive access is
# prevented by confinement, in four independent layers:
#   1. the per-key authorized_keys forced command (the sole SSH dispatcher)
#   2. vloud-git-shell's strict parser, which never evaluates the client's
#      command through a shell
#   3. the `Match User git` restrictions in /etc/ssh/sshd_config.d/60-vloud-vcs.conf
#   4. key-only access: the account has no password
if ! getent group git >/dev/null 2>&1; then
  groupadd --system git
  ok "created git system group"
fi
if ! id -u git >/dev/null 2>&1; then
  useradd --system --home-dir /home/git --create-home --shell /bin/sh --gid git git
  ok "created git service user (shell /bin/sh — see the note above)"
else
  ok "git service user exists"
fi
# Key-only: lock the password so there is no non-key authentication path.
passwd -l git >/dev/null 2>&1 || true

install -d -m 0700 -o git -g git /home/git/.ssh
touch /home/git/.ssh/authorized_keys
chown git:git /home/git/.ssh/authorized_keys
chmod 0600 /home/git/.ssh/authorized_keys

# Persistent Git root (plan D10). Deliberately under /var/lib/vloud and NOT
# under /opt/vloud: the upgrade flow moves the install dir aside, copies only
# vloud.db forward, prunes rollback trees to two, and rm -rf's the install dir
# on rollback — so a Git root inside it would be orphaned on upgrade and
# destroyed by the third one, while the database still listed every repository.
install -d -m 0750 -o root -g git /var/lib/vloud/git
# .locks and the .spool ROOT are 0750 with NO group write: on Unix the
# permission to unlink or rename an entry comes from write+execute on its
# DIRECTORY, so a group-writable .spool would let the git identity remove
# capacity.ledger.lock despite its own 0660 mode.
install -d -m 0750 -o root -g git /var/lib/vloud/git/.locks
install -d -m 0750 -o root -g git /var/lib/vloud/git/.spool
install -d -m 0750 -o root -g git /var/lib/vloud/git/.trash
install -d -m 0750 -o root -g git /var/lib/vloud/git/.tmp
# The two per-push DATA directories ARE group-writable: the SSH receive owner
# runs as git and genuinely must create its own <push-id>.live and
# <push-id>.json inside them.
install -d -m 0770 -o root -g git /var/lib/vloud/git/.spool/live
install -d -m 0770 -o root -g git /var/lib/vloud/git/.spool/post-receive
install -d -m 0750 -o root -g git /etc/vloud/vcs-hooks
install -d -m 0755 -o root -g root /usr/local/lib/vloud-git-hooks

# ---------------------------------------------------------------------------
# Pipeline execution account (vloud-ci).
#
# Pipeline steps execute code from tenant repositories, and on a pull-request
# run that code may be written by anyone who can open one. It therefore gets a
# THIRD identity, distinct from both root and the git account:
#
#   * not root — the engine is User=root with NoNewPrivileges=false and
#     EnvironmentFile=/etc/vloud.env; a step inheriting that is a host takeover
#     plus every credential in that file.
#   * not git — that account owns EVERY tenant's bare repository under
#     /var/lib/vloud/git. A build running as git could read and rewrite other
#     accounts' source.
#
# This account owns nothing but its own per-run workspaces. It gets nologin
# (unlike the git account, which needs a real shell for the SSH forced command)
# because nothing ever logs in as it — the engine spawns its processes directly
# through setpriv.
#
# If this account is missing, the engine REFUSES to run pipeline steps rather
# than falling back to root. That refusal is the design, not a failure mode to
# work around.
if ! getent group vloud-ci >/dev/null 2>&1; then
  groupadd --system vloud-ci
  ok "created vloud-ci system group"
fi
if ! id -u vloud-ci >/dev/null 2>&1; then
  useradd --system --no-create-home --home-dir /nonexistent \
    --shell /usr/sbin/nologin --gid vloud-ci vloud-ci
  ok "created vloud-ci pipeline execution user"
else
  ok "vloud-ci pipeline execution user exists"
fi
passwd -l vloud-ci >/dev/null 2>&1 || true

# Workspace root. 0711 on the root itself: the CI account must be able to
# TRAVERSE into its own run directory but must not be able to LIST the root and
# discover other runs. Each run directory below it is 0700 and owned by the CI
# account only for the lifetime of that run.
install -d -m 0711 -o root -g root /var/lib/vloud/pipeline-workspaces

# Artifact and release roots for the external-repo CI/CD feature.
#
# Under /var/lib/vloud and NOT under $VLOUD_INSTALL_DIR: the upgrade flow moves
# the install directory aside and prunes rollback trees, so anything stored
# there is destroyed by the third upgrade. That is the same reasoning that moved
# the Git root, and it applies identically to build artifacts and to the release
# directories a live application is being served from.
#
# 0750 root:root — the artifact root is ENGINE-OWNED. `vloud-ci` must not be
# able to read another account's build output, and it has no business writing
# into either root at all: the engine copies artifacts out of the workspace
# AFTER the step completes.
install -d -m 0750 -o root -g root /var/lib/vloud/artifacts
install -d -m 0751 -o root -g root /var/lib/vloud/apps

# OPTIONAL: the ASP.NET Core runtime (--with-dotnet).
#
# Runtime only, never the SDK: a build happens in the pipeline (which may well
# use a container or a prebuilt artifact), and what a HOST needs in order to
# serve `dotnet MyApp.dll` is the runtime. Installing the SDK here would add
# gigabytes for a capability the deploy path does not use.
if [[ "$VLOUD_WITH_DOTNET" == "1" ]]; then
  say "installing the ASP.NET Core runtime (--with-dotnet)"
  case "$OS_FAMILY" in
    debian)
      # Debian/Ubuntu ship dotnet in the distro repositories from Debian 12 /
      # Ubuntu 22.04 onward, so no Microsoft feed is added — one fewer external
      # trust root on a control-plane host.
      install_packages aspnetcore-runtime-8.0 || \
        install_packages dotnet-runtime-8.0 || \
        warn "could not install the ASP.NET Core runtime from the distribution repositories"
      ;;
    rhel)
      install_packages aspnetcore-runtime-8.0 || \
        warn "could not install the ASP.NET Core runtime from the distribution repositories"
      ;;
  esac
  if command -v dotnet >/dev/null 2>&1; then
    ok "dotnet present: $(dotnet --version 2>/dev/null || echo 'version unknown')"
  else
    warn "dotnet is still not on PATH — .NET applications will fail their deploy preflight"
  fi
fi

# Trial bootstrap — once-per-install set of values.
# /etc/machine-id is the canonical Linux machine identity since systemd ~2014.
if [[ -f /etc/machine-id ]] && [[ -s /etc/machine-id ]]; then
  RAW_MACHINE_ID=$(cat /etc/machine-id)
else
  RAW_MACHINE_ID=$(head -c 32 /dev/urandom | sha256sum | cut -d' ' -f1)
fi
# Same prefix-and-truncate the engine uses, so env and DB-side machine_id
# agree from the very first call.
MACHINE_ID=$(printf 'vloud:%s' "$RAW_MACHINE_ID" | sha256sum | cut -d' ' -f1 | head -c 32)

ENV_TMP=$(mktemp /etc/vloud.env.XXXXXX)
trap 'rm -f "$ENV_TMP"' EXIT

# 0.6.0 — cloud license sync. Engine registers itself with the
# license-server + pulls its trial/license. Without this URL set
# correctly, the heartbeat client defaults to http://127.0.0.1:3000
# (a local stub) and the engine never reaches the cloud.
#
# 0.6.0a fix: the URL is the LICENSE-SERVER hostname
# (license.vloud.app), NOT the admin-dashboard hostname
# (adminpanel.vloud.app). Pre-fix installs pointed at the admin
# dashboard and registration silently swallowed every POST as a
# SPA-HTML 200 because adminpanel.vloud.app has no /v1/* route.
#
# 0.6.0c: VLOUD_LICENSE_SERVER_URL_DEFAULT is now hoisted to the
# top of the script alongside VLOUD_INSTALL_DIR etc., so that
# `do_reconcile_env_file()` (reachable from --upgrade and --repair
# via the early mode dispatcher) can reference it without
# tripping `set -u`. Reference here kept as documentation.
: "${VLOUD_LICENSE_SERVER_URL_DEFAULT:?must be set in top-of-script defaults}"

if [[ ! -f "$ENV_FILE" ]]; then
  # First-time install — fresh env file with trial values.
  INSTALL_ID=$(head -c 16 /dev/urandom | xxd -p)
  TRIAL_EXPIRES_AT=$(date -u -d "+${VLOUD_TRIAL_DAYS} days" +%Y-%m-%dT%H:%M:%SZ)
  cat > "$ENV_TMP" <<EOF
# Vloud engine environment.  Edit-don't-edit: changes to VLOUD_MACHINE_ID
# / VLOUD_INSTALL_ID / VLOUD_TRIAL_EXPIRES_AT without recomputing the HMAC
# in trial_state will mark the install as tampered.
PORT=$VLOUD_PORT
NODE_ENV=production
VLOUD_BIND_HOST=$VLOUD_BIND_HOST
VLOUD_JWT_SECRET=$(openssl rand -hex 64)
# 1, not 0. The engine runs as User=vloud (uid 997), so it has no
# privilege of its own; with this at 0 privilegedMode() returns
# "deferred" and every daemon bootstrap the wizard tries to run is
# refused with "no privilege to install (running as uid=997 without
# VLOUD_SUDO_FALLBACK)". The sudoers grant it needs is installed a few
# lines below — this just tells the engine it may use it. The upgrade
# path already upserts 1; fresh install disagreed.
VLOUD_SUDO_FALLBACK=1
VLOUD_MACHINE_ID=$MACHINE_ID
VLOUD_INSTALL_ID=$INSTALL_ID
VLOUD_TRIAL_EXPIRES_AT=$TRIAL_EXPIRES_AT
# 0.6.0: cloud license-server URL. Registration + license-pull workers
# POST/GET against this. To disable cloud sync entirely (offline mode),
# set VLOUD_REGISTRATION_DISABLE=1 and VLOUD_LICENSE_PULL_DISABLE=1
# below — the engine will continue to honour a manually-installed
# /var/lib/vloud/license.dat.
VLOUD_LICENSE_SERVER_URL=$VLOUD_LICENSE_SERVER_URL_DEFAULT
# PHP-FPM version actually installed on this host. The engine builds
# service names (php<v>-fpm) and pool paths (/etc/php/<v>/fpm/…) from
# it. Set because it is NOT always 8.3 — Ubuntu 26.04 ships 8.5 and
# has no php8.3 packages, so an assumed 8.3 would name a unit and a
# directory that do not exist.
VLOUD_PHP_VERSION=$PHP_VERSION
# Which sudo the engine shells out to. Ubuntu 26.04's default sudo is
# sudo-rs, which cannot parse Vloud's wildcard-argument sudoers rules;
# classic sudo ships there as /usr/bin/sudo.ws. Must match the visudo
# used to validate the fragments — see scripts/_os-family.sh.
VLOUD_SUDO_BIN=$SUDO_BIN
EOF
  if [[ "$COEXIST" -eq 1 ]]; then
    echo "VLOUD_COEXIST=1" >> "$ENV_TMP"
    echo "VLOUD_SAFE_COEXIST=1" >> "$ENV_TMP"
    echo "VLOUD_CUTOVER_MODE=0" >> "$ENV_TMP"
  fi
  install -m 0640 -o root -g vloud "$ENV_TMP" "$ENV_FILE"
  rm -f "$ENV_TMP"
  ok "wrote fresh $ENV_FILE (trial $VLOUD_TRIAL_DAYS days, cloud sync → adminpanel)"
else
  # Re-run path — preserve existing values, only fill gaps.
  cp "$ENV_FILE" "$ENV_TMP"
  upsert() {
    local k="$1" v="$2"
    if grep -q "^$k=" "$ENV_TMP"; then return; fi
    printf '%s=%s\n' "$k" "$v" >> "$ENV_TMP"
  }
  upsert PORT "$VLOUD_PORT"
  upsert NODE_ENV production
  upsert VLOUD_BIND_HOST "$VLOUD_BIND_HOST"
  upsert VLOUD_JWT_SECRET "$(openssl rand -hex 64)"
  upsert VLOUD_SUDO_FALLBACK 1
  upsert VLOUD_MACHINE_ID "$MACHINE_ID"
  upsert VLOUD_INSTALL_ID "$(head -c 16 /dev/urandom | xxd -p)"
  upsert VLOUD_TRIAL_EXPIRES_AT "$(date -u -d "+${VLOUD_TRIAL_DAYS} days" +%Y-%m-%dT%H:%M:%SZ)"
  # Upgrades from before PHP detection have no VLOUD_PHP_VERSION. The
  # engine defaults to 8.3 without it, which is right for the hosts
  # that predate this and wrong for any host where 8.3 was never
  # available — fill it in with what is actually installed.
  upsert VLOUD_PHP_VERSION "$PHP_VERSION"
  upsert VLOUD_SUDO_BIN "$SUDO_BIN"
  if [[ "$COEXIST" -eq 1 ]]; then
    upsert VLOUD_COEXIST 1
    upsert VLOUD_SAFE_COEXIST 1
    upsert VLOUD_CUTOVER_MODE 0
  fi

  # 0.6.0: upsert VLOUD_LICENSE_SERVER_URL. Three rewrite cases:
  #
  #   1. Localhost (pre-0.6 default — engine never reached cloud).
  #   2. adminpanel.vloud.app (0.6.0 default — but adminpanel doesn't
  #      proxy /v1/* to the license-server; that was the 0.6.0a bug).
  #   3. Missing → just upsert.
  #
  # Rewriting case 2 in place is critical: every 0.6.0 install
  # bootstrap'd with adminpanel.vloud.app, and those engines never
  # successfully registered. The upgrade path corrects them.
  if grep -qE '^VLOUD_LICENSE_SERVER_URL=https?://(127\.0\.0\.1|localhost|adminpanel\.vloud\.app)' "$ENV_TMP"; then
    sed -i -E "s|^VLOUD_LICENSE_SERVER_URL=.*|VLOUD_LICENSE_SERVER_URL=$VLOUD_LICENSE_SERVER_URL_DEFAULT|" "$ENV_TMP"
    ok "rewrote VLOUD_LICENSE_SERVER_URL → $VLOUD_LICENSE_SERVER_URL_DEFAULT"
  else
    upsert VLOUD_LICENSE_SERVER_URL "$VLOUD_LICENSE_SERVER_URL_DEFAULT"
  fi

  install -m 0640 -o root -g vloud "$ENV_TMP" "$ENV_FILE"
  rm -f "$ENV_TMP"
  ok "updated $ENV_FILE (preserving existing values; cloud sync → adminpanel)"
fi

# Source values for the summary later.
TRIAL_EXPIRES_AT=$(grep -oE '^VLOUD_TRIAL_EXPIRES_AT=.*' "$ENV_FILE" | cut -d= -f2-)

# Vloud-managed dirs.
#
# 2026-05-14: /var/lib/vloud (the PARENT) is owned by vloud:vloud
# because the engine writes state files directly there at runtime:
# .upgrade-progress (PR1 JSONL progress; bootstrap.sh writes as root,
# engine reads as vloud), license.dat, .tamper-events, .config-ok.
# bootstrap.sh's earlier (pre-2026-05-14) bug created this as
# root:root, which silently broke the engine's state writes; the
# do_verify_dependencies gate catches it now ("/var/lib/vloud not
# writable by user vloud"). Sub-directories below can be root-owned
# safely because the engine doesn't traverse into them — backups +
# crontabs are write-by-root-only (bootstrap.sh's do_backup_state +
# operator-provisioned cron).
# 0751, not 0750 — see the note at the first install of this directory: the
# `git` service account must be able to traverse here to reach the root:git
# Git root, and it is in no group that owns this path.
install -d -m 0751 -o vloud -g vloud /var/lib/vloud
install -d -m 0750 -o root  -g root  /var/lib/vloud/crontabs /var/lib/vloud/backups
install -d -m 0750 -o root  -g root  /etc/vloud /etc/vloud/ssl /etc/vloud/certs
install -d -m 0750 -o vloud -g vloud /var/lib/vloud/staging \
  /var/lib/vloud/staging/nginx /var/lib/vloud/staging/php \
  /var/lib/vloud/staging/ssl /var/lib/vloud/staging/fs \
  /var/lib/vloud/staging/mail /var/lib/vloud/staging/installers
install -d -m 0755 -o root -g "$HTTP_GROUP" /var/lib/vloud/acme \
  /var/lib/vloud/acme/.well-known /var/lib/vloud/acme/.well-known/acme-challenge
ok "/var/lib/vloud/* + /etc/vloud/* directories ready"

# 0.6.0a — fetch the active license-signing public key and install
# it at /etc/vloud/license-v1.pub.pem. The engine's license-pull
# worker uses this PEM to verify JWS issued by the cloud. Without
# it, every pull is rejected as "signature/shape mismatch" and the
# sidebar stays on "Pulling license…" forever.
#
# Idempotent: re-runs on upgrade fetch a fresh copy (the signing
# key rotates rarely, but when it does this is how customers pick
# up the new public counterpart without a manual scp).
LICENSE_PUBKEY_PATH=/etc/vloud/license-v1.pub.pem
LICENSE_PUBKEY_URL="${VLOUD_LICENSE_SERVER_URL_DEFAULT}/v1/license-pubkey"
say "fetching license-signing public key from $LICENSE_PUBKEY_URL"
TMP_PEM=$(mktemp /tmp/vloud-license-pubkey.XXXXXX.pem)
if curl -fsSL --max-time 15 -o "$TMP_PEM" "$LICENSE_PUBKEY_URL" \
   && grep -q 'BEGIN PUBLIC KEY' "$TMP_PEM"; then
  install -m 0644 -o root -g root "$TMP_PEM" "$LICENSE_PUBKEY_PATH"
  ok "$LICENSE_PUBKEY_PATH installed ($(wc -c < "$LICENSE_PUBKEY_PATH") bytes)"
else
  warn "could not fetch license public key — engine will retry on first pull but trial sync will fail until reachable"
fi
rm -f "$TMP_PEM"

# Sudoers fragment so the engine can run privileged verbs.  We DO this here
# so the bootstrap is fully end-to-end; the operator can drop the file
# afterwards if they want a more locked-down policy.
if [[ -f "$VLOUD_INSTALL_DIR/packages/vloud-agent/install/sudoers.d/vloud" ]]; then
  STAGE=$(mktemp /tmp/vloud-sudoers.XXXXXX)
  trap 'rm -f "$STAGE"' EXIT
  sed 's/__VLOUD_USER__/vloud/g' \
    "$VLOUD_INSTALL_DIR/packages/vloud-agent/install/sudoers.d/vloud" > "$STAGE"
  # Validate with the sudo implementation we actually chose — see
  # detect_sudo_impl. On 26.04 the canonical `visudo` is sudo-rs, which
  # rejects syntax classic sudo accepts.
  if "${VISUDO_BIN:-visudo}" -c -f "$STAGE" >/dev/null; then
    install -m 0440 -o root -g root "$STAGE" /etc/sudoers.d/vloud
    ok "sudoers fragment installed"
  else
    warn "sudoers template failed visudo -c — skipping"
  fi
  rm -f "$STAGE"
fi

# Webmail provisioner helpers → /usr/local/sbin. The engine shells out to
# these (via the scoped NOPASSWD grants above) when the operator clicks
# "Install webmail". They ship in the release tree as scripts/vloud-webmail-*.sh;
# install them under the exact names the engine + sudoers expect (no .sh).
# Root-owned + 0755 so a non-root engine can execute but not modify them.
for _wm in install install-sogo uninstall; do
  _src="$VLOUD_INSTALL_DIR/scripts/vloud-webmail-${_wm}.sh"
  if [[ -f "$_src" ]]; then
    install -m 0755 -o root -g root "$_src" "/usr/local/sbin/vloud-webmail-${_wm}"
  fi
done
ok "webmail provisioner helpers installed to /usr/local/sbin"

# Caddy-fronted domain provisioner → /usr/local/sbin. The engine shells out
# to this (via the scoped NOPASSWD grant) to serve user domains through Caddy
# on hosts where Caddy owns :80/:443. Same install convention as webmail.
_dom_src="$VLOUD_INSTALL_DIR/scripts/vloud-domain-apply.sh"
if [[ -f "$_dom_src" ]]; then
  install -m 0755 -o root -g root "$_dom_src" "/usr/local/sbin/vloud-domain-apply"
  ok "caddy domain provisioner installed to /usr/local/sbin"
fi

# Ownership of the engine tree — must match the unit's User=, because
# the engine drops CAP_DAC_OVERRIDE and so gets no permission bypass
# from being uid 0. Hardcoding `vloud` here while packaging/systemd
# ships User=root produced a fresh install whose engine could not write
# its own database (SQLITE_READONLY → HTTP 500 on /api/setup/state).
# We do NOT chown /opt/vloud as a whole because the operator may have
# other things there; the packages/ subtree only.
ENGINE_USER=$(engine_service_user)
chown -R "$ENGINE_USER:$ENGINE_USER" "$VLOUD_INSTALL_DIR/packages" "$VLOUD_INSTALL_DIR/node_modules" 2>/dev/null || true
ok "engine tree owned by $ENGINE_USER:$ENGINE_USER"

# ---------------------------------------------------------------------------
# Native VCS helpers (plan D37, D11a, Task 21/25).
#
# These live OUTSIDE the install dir on purpose. /opt/vloud is moved aside and
# replaced by every upgrade, and rm -rf'd on rollback — a helper installed
# inside it would vanish mid-upgrade while sshd was still configured to execute
# it, which is a broken SSH surface rather than a degraded one.
#
# All three are root-owned and NOT writable by the `git` account: the account
# they confine must never be able to replace the program that confines it.
# vloud-vcs-lock-supervisor is deliberately NOT setuid — sshd runs the forced
# command as `git`, and every permission decision in the lock protocol follows
# from the whole chain carrying git's credentials.
install -d -m 0755 -o root -g root /usr/local/libexec /usr/local/lib/vloud-git-hooks

VCS_SRC="$VLOUD_INSTALL_DIR/packages/server"

# The lock supervisor is compiled at release-build time; fall back to compiling
# here if the artifact shipped sources only.
if [[ -f "$VCS_SRC/native/vloud-vcs-lock-supervisor/vloud-vcs-lock-supervisor" ]]; then
  install -m 0755 -o root -g root \
    "$VCS_SRC/native/vloud-vcs-lock-supervisor/vloud-vcs-lock-supervisor" \
    /usr/local/libexec/vloud-vcs-lock-supervisor
  ok "installed vloud-vcs-lock-supervisor"
elif [[ -f "$VCS_SRC/native/vloud-vcs-lock-supervisor/vloud-vcs-lock-supervisor.c" ]] && command -v cc >/dev/null 2>&1; then
  if make -C "$VCS_SRC/native/vloud-vcs-lock-supervisor" >/dev/null 2>&1; then
    install -m 0755 -o root -g root \
      "$VCS_SRC/native/vloud-vcs-lock-supervisor/vloud-vcs-lock-supervisor" \
      /usr/local/libexec/vloud-vcs-lock-supervisor
    ok "compiled and installed vloud-vcs-lock-supervisor"
  else
    warn "could not build vloud-vcs-lock-supervisor — SSH Git access will stay disabled (Gate A false)"
  fi
else
  warn "vloud-vcs-lock-supervisor not present — SSH Git access will stay disabled (Gate A false)"
fi

# vloud-import-contain: the Azure-import child-subtree containment helper.
#
# It lives OUTSIDE the install dir for the same reason as the lock supervisor:
# /opt/vloud is moved aside by every upgrade and rm -rf'd on rollback, and a
# helper that vanished mid-upgrade while an import held live Git children would
# be a broken containment surface rather than a degraded one.
#
# root:root 0755 and deliberately NOT setuid. It runs as the job worker's
# identity and drops each spawned Git child to the `git` account itself, so it
# needs no elevated file permissions of its own — and the `git` account must not
# be able to write the program that confines it.
install -d -m 0755 -o root -g root /usr/local/libexec/vloud
if [[ -f "$VCS_SRC/native/vloud-import-contain/vloud-import-contain" ]]; then
  install -m 0755 -o root -g root \
    "$VCS_SRC/native/vloud-import-contain/vloud-import-contain" \
    /usr/local/libexec/vloud/vloud-import-contain
  ok "installed vloud-import-contain"
elif [[ -f "$VCS_SRC/native/vloud-import-contain/vloud-import-contain.c" ]] && command -v cc >/dev/null 2>&1; then
  if make -C "$VCS_SRC/native/vloud-import-contain" >/dev/null 2>&1; then
    install -m 0755 -o root -g root \
      "$VCS_SRC/native/vloud-import-contain/vloud-import-contain" \
      /usr/local/libexec/vloud/vloud-import-contain
    ok "compiled and installed vloud-import-contain"
  else
    warn "could not build vloud-import-contain — Azure DevOps import will stay closed (I.G10)"
  fi
else
  warn "vloud-import-contain not present — Azure DevOps import will stay closed (I.G10)"
fi

# vloud-git-shell: the per-key forced command target. A tiny wrapper so the
# authorized_keys line names a stable absolute path that does not move with the
# engine tree.
if [[ -f "$VCS_SRC/dist/services/vcs/git-shell.js" ]]; then
  cat > /usr/local/sbin/vloud-git-shell <<'VLOUD_GIT_SHELL_EOF'
#!/bin/sh
# Vloud Native VCS SSH forced-command helper. Generated by the installer.
#
# This program owns ZERO leases and creates ZERO durable state: it parses the
# client's request strictly, authorizes it over the private Unix socket, and
# starts vloud-vcs-lock-supervisor, which is the sole transport owner.
exec /usr/bin/node /opt/vloud/packages/server/dist/services/vcs/git-shell.js "$@"
VLOUD_GIT_SHELL_EOF
  chown root:root /usr/local/sbin/vloud-git-shell
  chmod 0755 /usr/local/sbin/vloud-git-shell
  ok "installed vloud-git-shell"
fi

# The shared receive hooks are rendered by the engine at boot (they embed the
# resolved node + runner paths), so only the directory is prepared here.
chown root:root /usr/local/lib/vloud-git-hooks
chmod 0755 /usr/local/lib/vloud-git-hooks

# A3 (2026-05-10) — logrotate config for non-journald log paths.
# journald rotates the engine's stdout/stderr automatically via its own
# config; this file covers /var/log/vloud/*.log, the modsec audit log,
# and per-domain nginx access/error logs.
say "installing logrotate config"
if [[ -f "$VLOUD_INSTALL_DIR/packaging/logrotate/vloud" ]]; then
  install -m 0644 -o root -g root \
    "$VLOUD_INSTALL_DIR/packaging/logrotate/vloud" /etc/logrotate.d/vloud
  ok "/etc/logrotate.d/vloud installed"
else
  warn "packaging/logrotate/vloud missing in install tree; skipping"
fi
mkdir -p /var/log/vloud
chown vloud:vloud /var/log/vloud

# 0.7.0 architecture: VLoud does NOT claim port 80 or default_server.
#
# The engine binds to 127.0.0.1:$VLOUD_PORT only. nginx is a gateway
# that the operator configures with a panel domain during onboarding.
# Initial onboarding is accessed directly at http://SERVER-IP:$VLOUD_PORT.
#
# Migration from pre-0.7.0: if vloud-default exists, remove it.
if [[ -f /etc/nginx/sites-enabled/vloud-default ]]; then
  say "migrating: removing vloud-default (VLoud no longer claims port 80/default_server)"
  rm -f /etc/nginx/sites-enabled/vloud-default
  if nginx -t 2>&1 | grep -E 'test is successful|syntax is ok' >/dev/null; then
    systemctl reload nginx 2>/dev/null || true
  fi
  ok "vloud-default removed"
fi
# Only reachable at the server's own address when the engine is bound
# to all interfaces; by default it is loopback-only. The final summary
# prints the working URL and how to reach it remotely.
if [[ "$VLOUD_BIND_HOST" == "0.0.0.0" || "$VLOUD_BIND_HOST" == "::" ]]; then
  say "VLoud panel accessible at http://<server-ip>:${VLOUD_PORT}"
else
  say "VLoud panel bound to ${VLOUD_BIND_HOST}:${VLOUD_PORT} (loopback-only by default)"
fi

# systemd units — D5. Three services (engine + worker + scheduler)
# under one slice. Canonical sources live in
# packaging/systemd/*.service in the engine repo; we template
# WorkingDirectory + Port here because $VLOUD_INSTALL_DIR /
# $VLOUD_PORT can differ per install. Matching changes to the
# canonical files MUST be reflected here (no shared templater
# yet — the indirection wasn't worth a sed pass).
say "installing systemd units"

cat > "$SYSTEMD_SLICE" <<EOF
[Unit]
Description=Vloud control plane (engine + workers)
Documentation=https://docs.vloud.sh/
DefaultDependencies=true
Before=vloud.service vloud-job-worker.service vloud-scheduler.service

[Slice]
EOF

cat > "$SYSTEMD_UNIT" <<EOF
[Unit]
Description=Vloud control plane (Fastify on $VLOUD_PORT)
Documentation=https://docs.vloud.sh/
After=network-online.target redis.service
Wants=network-online.target redis.service
StartLimitBurst=5
StartLimitIntervalSec=60s

[Service]
# Type=notify (A1, 2026-05-10) — engine boots through migrations +
# integrity chain + listen before ready. Mirrors
# packaging/systemd/vloud.service; keep these two in sync.
# NotifyAccess=all (not =main): the engine's sd-notify implementation
# shells out to the `systemd-notify` binary (Node has no native
# AF_UNIX_DGRAM); the spawned binary is a child pid so =main rejects
# every READY=1. =all accepts notifies from any process in the unit's
# cgroup. Switch back to =main when sd-notify is rewritten in pure JS.
Type=notify
NotifyAccess=all
TimeoutStartSec=120s
WatchdogSec=60s
User=root
Group=root
Slice=vloud.slice
WorkingDirectory=$(if [[ "${VLOUD_ENGINE_FORMAT:-sea}" == "sea" ]]; then echo "$VLOUD_INSTALL_DIR"; else echo "$VLOUD_INSTALL_DIR/packages/server"; fi)
EnvironmentFile=$ENV_FILE
ExecStart=$(if [[ "${VLOUD_ENGINE_FORMAT:-sea}" == "sea" ]]; then echo "$VLOUD_INSTALL_DIR/vloud-engine"; else echo "/usr/bin/node dist/index.js"; fi)

Restart=on-failure
RestartSec=3s
TimeoutStopSec=30s
KillSignal=SIGTERM
SendSIGKILL=yes
# KillMode=process (NOT the default control-group): engine-driven
# upgrades spawn bootstrap.sh --upgrade as a child of this unit, and
# bootstrap.sh do_stop_services() runs `systemctl stop vloud` mid-way
# to swap the tree. With control-group, that stop reaps the whole
# cgroup — including the running upgrade — so the swap never completes
# and the dashboard hangs on "Installing…". KillMode=process signals
# only the engine's main PID on stop, letting the upgrade child live
# to finish the swap and restart the engine. Keep in sync with
# packaging/systemd/vloud.service.
KillMode=process

# Phase E (2026-05-14): when the engine repeatedly fails to start
# (StartLimitBurst hit), trigger an automatic rollback via the
# vloud-rollback.service unit. The unit shells to
# bootstrap.sh --rollback --automatic, which restores the newest
# /opt/vloud.pre-upgrade-<ts>/ slot. Operator can suppress by
# touching /var/lib/vloud/.rollback-disabled (debugging mode).
StartLimitIntervalSec=120
StartLimitBurst=5
OnFailure=vloud-rollback.service

StandardOutput=journal
StandardError=journal
SyslogIdentifier=vloud

NoNewPrivileges=false
ProtectSystem=false
ProtectHome=false
# PrivateTmp=false (NOT true): engine-driven upgrades spawn bootstrap.sh
# as a child of this unit and it must OUTLIVE `systemctl stop vloud` to
# finish the swap (see KillMode=process above). With PrivateTmp=true,
# systemd destroys the service's private /tmp on stop, and the still-
# running upgrade then fails its next mktemp /tmp/... with ENOENT under
# `set -e` — dying right before it restarts the engine. Keep in sync
# with packaging/systemd/vloud.service.
PrivateTmp=false
ProtectKernelTunables=true
ProtectKernelModules=true
ProtectControlGroups=true
RestrictRealtime=true
ReadWritePaths=/var/lib/vloud /var/log/vloud
# Native VCS runtime directory (plan gate 0.G5 / §3.1).
#
# /run/vloud-vcs holds the cross-process slot files and the private VCS
# Unix socket. Both are NON-PERSISTENT by design: a slot file that survived
# a reboot while its lease did not would void mutual exclusion, so systemd
# recreates the directory empty at every boot and the engine creates the
# slot files once, before the write path is served.
#
# NOTE: this directive must exist in BOTH packaging/systemd/vloud.service
# and the bootstrap.sh heredoc unit — bootstrap writes its own units to
# /etc/systemd/system, so a directive added to only one will not reach a
# fresh install.
RuntimeDirectory=vloud-vcs
RuntimeDirectoryMode=0750
AmbientCapabilities=CAP_NET_BIND_SERVICE
# The engine runs as User=root (the intended model — see the long note in
# packaging/systemd/vloud.service). uid 0 alone grants nothing here: the
# CapabilityBoundingSet is what decides what root may do, and it must be
# wide enough for the daemon-provisioning the engine shells out to. An
# earlier fresh-install unit ran as User=vloud with a narrow set
# (NET_BIND_SERVICE SETUID SETGID); that broke mail-domain provisioning
# two ways — root-via-sudo couldn't traverse the vloud-owned 0750
# /var/lib/vloud to read staged DKIM keys (needs CAP_DAC_OVERRIDE), and
# the vloud+sudo model needs complete sudoers grants for EVERY privileged
# command (the DKIM chown/chmod/reload/postmap were ungranted → denied).
# User=root + direct-exec sidesteps the sudoers-completeness problem.
# Keep this list in sync with packaging/systemd/vloud.service.
CapabilityBoundingSet=CAP_CHOWN CAP_DAC_OVERRIDE CAP_DAC_READ_SEARCH CAP_FOWNER CAP_FSETID CAP_KILL CAP_SETGID CAP_SETUID CAP_SETPCAP CAP_NET_BIND_SERVICE CAP_NET_ADMIN CAP_SYS_ADMIN CAP_SYS_RESOURCE CAP_AUDIT_WRITE CAP_MKNOD

[Install]
WantedBy=multi-user.target
EOF

cat > "$SYSTEMD_WORKER" <<EOF
[Unit]
Description=Vloud job worker (BullMQ consumer for deploys + SSL)
Documentation=https://docs.vloud.sh/
After=network-online.target redis-server.service redis.service vloud.service
Wants=network-online.target redis-server.service redis.service
PartOf=vloud.service
StartLimitBurst=5
StartLimitIntervalSec=60s

[Service]
Type=simple
User=root
Group=root
Slice=vloud.slice
WorkingDirectory=$VLOUD_INSTALL_DIR/packages/server
EnvironmentFile=$ENV_FILE
ExecStart=/usr/bin/node dist/workers/job-worker.js

Restart=on-failure
RestartSec=3s
TimeoutStartSec=30s
TimeoutStopSec=120s
KillSignal=SIGTERM
SendSIGKILL=yes

StandardOutput=journal
StandardError=journal
SyslogIdentifier=vloud-job-worker

NoNewPrivileges=false
ProtectSystem=false
ProtectHome=false
PrivateTmp=true
ProtectKernelTunables=true
ProtectKernelModules=true
ProtectControlGroups=true
RestrictRealtime=true
ReadWritePaths=/var/lib/vloud /var/log/vloud

[Install]
WantedBy=multi-user.target
EOF

cat > "$SYSTEMD_SCHEDULER" <<EOF
[Unit]
Description=Vloud scheduler (periodic job enqueuer + sweepers)
Documentation=https://docs.vloud.sh/
After=network-online.target redis-server.service redis.service vloud-job-worker.service
Wants=network-online.target redis-server.service redis.service
PartOf=vloud.service
StartLimitBurst=5
StartLimitIntervalSec=60s

[Service]
Type=simple
User=root
Group=root
Slice=vloud.slice
WorkingDirectory=$VLOUD_INSTALL_DIR/packages/server
EnvironmentFile=$ENV_FILE
ExecStart=/usr/bin/node dist/workers/scheduler.js

Restart=on-failure
RestartSec=3s
TimeoutStartSec=30s
TimeoutStopSec=30s
KillSignal=SIGTERM
SendSIGKILL=yes

StandardOutput=journal
StandardError=journal
SyslogIdentifier=vloud-scheduler

NoNewPrivileges=false
ProtectSystem=false
ProtectHome=false
PrivateTmp=true
ProtectKernelTunables=true
ProtectKernelModules=true
ProtectControlGroups=true
RestrictRealtime=true
ReadWritePaths=/var/lib/vloud /var/log/vloud

[Install]
WantedBy=multi-user.target
EOF

# Phase E (2026-05-14): vloud-rollback.service — triggered by
# vloud.service's OnFailure hook when the engine crash-loops past
# StartLimitBurst. Shells to bootstrap.sh --rollback --automatic
# which restores /opt/vloud.pre-upgrade-<ts>/ (the newest slot).
#
# The unit guards on the absence of /var/lib/vloud/.rollback-disabled
# so a debugging operator can suppress auto-rollback by touching that
# file. Re-enable by deleting it.
#
# Type=oneshot — fires, does its work, exits. Not persistent.
cat > "$SYSTEMD_ROLLBACK" <<EOF
[Unit]
Description=Vloud automatic rollback on repeated engine start-failure
Documentation=https://docs.vloud.sh/operations/install-upgrade-runbook.md
# Suppress when operator is debugging.
ConditionPathExists=!/var/lib/vloud/.rollback-disabled
# Avoid an infinite rollback loop: if rollback ITSELF starts firing
# repeatedly, stop trying after 3 attempts in 10 minutes.
StartLimitBurst=3
StartLimitIntervalSec=600

[Service]
Type=oneshot
User=root
# --automatic flag tells bootstrap.sh this is a programmatic invocation,
# not an operator click. (Currently identical to plain --rollback; the
# flag exists so operator-vs-systemd-driven rollbacks are
# distinguishable in the progress log.)
ExecStart=$VLOUD_INSTALL_DIR/scripts/bootstrap.sh --rollback --automatic

StandardOutput=journal
StandardError=journal
SyslogIdentifier=vloud-rollback
EOF

systemctl daemon-reload
systemctl enable vloud.service vloud-job-worker.service vloud-scheduler.service >/dev/null 2>&1
# vloud-rollback.service is NOT enabled (don't start at boot) — it's
# triggered only by vloud.service's OnFailure hook.
ok "vloud.{service,job-worker,scheduler,rollback} installed + enabled"

# Re-normalise ownership now that the units EXIST.
#
# The earlier chown runs before this point, so engine_service_user()
# had no installed unit to read and fell back to the one in the release
# tree (packaging/systemd/vloud.service → User=root). This block then
# installs bootstrap's own units, which use User=vloud. The result was
# a tree owned by root:root under a service running as vloud, and the
# engine refused to boot: "BOOT REFUSED — SQLite disk…".
#
# Reading the unit that is actually installed makes the two agree
# regardless of which template won.
ENGINE_USER=$(engine_service_user)
chown -R "$ENGINE_USER:$ENGINE_USER" \
  "$VLOUD_INSTALL_DIR/packages" "$VLOUD_INSTALL_DIR/node_modules" 2>/dev/null || true
ok "ownership re-normalised to $ENGINE_USER:$ENGINE_USER (matches the installed unit)"

# ─── 4b. SELinux + firewall prep (RHEL family; no-op on Debian) ───
# Must run BEFORE the engine starts: under enforcing SELinux the engine
# can't bind its port and nginx can't proxy to it without these.
configure_selinux
if [[ "$OS_FAMILY" == rhel ]]; then
  # nginx serves the panel on 80/443; the engine itself stays loopback.
  fw_open_port 80 tcp
  fw_open_port 443 tcp
  fw_reload
fi

# ─── 5. Start engine + workers ───
# Order: engine first, then worker, then scheduler. PartOf= on the
# workers means a `systemctl restart vloud` cycles all three;
# starting them individually here lets us surface failures
# distinctly (a worker boot loop won't get blamed on the engine).
say "starting engine"
systemctl restart vloud.service

HEALTHY=0
for i in $(seq 1 30); do
  if curl -fsS --max-time 1 -o /dev/null -w '' "http://127.0.0.1:$VLOUD_PORT/api/health" 2>/dev/null; then
    HEALTHY=1; break
  fi
  sleep 1
done
[[ "$HEALTHY" -eq 1 ]] || die "engine did not become healthy on :$VLOUD_PORT within 30s — see journalctl -u vloud" 4

ok "engine healthy on :$VLOUD_PORT"

# Start the worker + scheduler now that the engine is up. We do
# this best-effort: if Redis isn't running on this host the worker
# unit's Requires=redis.service will refuse to start, and that's
# the right failure mode (BullMQ has no Redis fallback). Operators
# on Redis-less installs can disable these units.
say "starting workers"
# 2026-05-14: Redis is now installed + verified in step 2, so this
# is fail-fast instead of the previous warn-and-skip. If the operator
# explicitly opted out via VLOUD_BOOTSTRAP_NO_REDIS=1 (offline-only
# install variant — undocumented escape hatch), the workers go up
# disabled and the engine logs a degraded-mode warning at boot.
if [[ "${VLOUD_BOOTSTRAP_NO_REDIS:-0}" == "1" ]]; then
  warn "VLOUD_BOOTSTRAP_NO_REDIS=1 — workers NOT started (BullMQ paths disabled)"
elif systemctl is-active --quiet redis-server.service 2>/dev/null \
     || systemctl is-active --quiet redis.service 2>/dev/null; then
  if systemctl restart vloud-job-worker.service vloud-scheduler.service 2>/dev/null; then
    ok "vloud-job-worker + vloud-scheduler running"
  else
    do_dump_failure_journal
    die "worker or scheduler failed to start — check 'journalctl -u vloud-job-worker' / 'journalctl -u vloud-scheduler'" 9
  fi
else
  die "redis is not active — bootstrap should have ensured this in step 2. Inspect 'systemctl status redis-server' + 'journalctl -u redis-server'" 9
fi

# ─── 6. Phase 1 daemon stack ───
# Postfix / Dovecot / Rspamd / fail2ban / BIND / pure-ftpd / nftables /
# tenant.slice templates / storage-quota tools. Each child script is
# idempotent and short-circuits if the daemon is already configured.
# cPanel-detected hosts skip the stack by default (cPanel owns those
# services). Master skip: VLOUD_BOOTSTRAP_NO_DAEMON_STACK=1.
say "configuring Phase 1 daemon stack"

DAEMON_STACK_SCRIPT="$VLOUD_INSTALL_DIR/scripts/bootstrap-daemon-stack.sh"
# Fallback when bootstrap.sh is run from the source tree (e.g. dev test
# of the bootstrap pipeline before a tarball ships).
if [[ ! -x "$DAEMON_STACK_SCRIPT" ]]; then
  ALT="$(dirname "${BASH_SOURCE[0]}")/bootstrap-daemon-stack.sh"
  [[ -x "$ALT" ]] && DAEMON_STACK_SCRIPT="$ALT"
fi

if [[ ! -x "$DAEMON_STACK_SCRIPT" ]]; then
  warn "bootstrap-daemon-stack.sh missing — daemons can be installed manually later"
else
  # We pass through the cPanel detection result; the child script will
  # also detect on its own (defence in depth).
  if [[ "$COEXIST" -eq 1 ]] && [[ "${VLOUD_BOOTSTRAP_DAEMON_STACK:-0}" != "1" ]]; then
    warn "cPanel detected — daemon stack skipped (set VLOUD_BOOTSTRAP_DAEMON_STACK=1 to override)"
  else
    DAEMON_RC=0
    bash "$DAEMON_STACK_SCRIPT" || DAEMON_RC=$?
    case "$DAEMON_RC" in
      0) ok "daemon stack: all installed" ;;
      2) warn "daemon stack: one or more daemons failed (continued; see log above)" ;;
      *) warn "daemon stack: aborted (rc=$DAEMON_RC)" ;;
    esac
  fi
fi

# 2026-05-14: Final dependency gate. Bootstrap is considered complete
# only when every required runtime dep is present + healthy + reachable.
# Fresh-host validation on 188.245.113.223 found that bootstrap was
# silently producing engines without redis; this gate prevents that
# from ever shipping again. Failure here is fatal — install summary
# below is meaningless if deps don't pass.
if ! do_verify_dependencies; then
  die "install verification failed — see failures above. Bootstrap did NOT complete cleanly." 10
fi

# Trial status read-back.
TRIAL_JSON=$(curl -fsS "http://127.0.0.1:$VLOUD_PORT/api/install-trial/status" 2>/dev/null || echo '{}')
TRIAL_STATE=$(printf '%s' "$TRIAL_JSON" | grep -oE '"state":"[^"]+"' | cut -d'"' -f4 || echo unknown)
TRIAL_DAYS_LEFT=$(printf '%s' "$TRIAL_JSON" | grep -oE '"days_remaining":[0-9]+' | cut -d: -f2 || echo "?")

# Public IP — hostname -I is reliable on cloud servers; ifconfig.io is the
# fallback for one-NIC nodes behind NAT.
PUBLIC_IP=$(hostname -I 2>/dev/null | awk '{print $1}')
if [[ -z "$PUBLIC_IP" ]] || [[ "$PUBLIC_IP" == "127."* ]]; then
  PUBLIC_IP=$(curl -fsS --max-time 5 https://ifconfig.io 2>/dev/null || echo "<unknown>")
fi

# ─── Summary ───
echo
printf "${GREEN}═══════════════════════════════════════════════════════════════${RESET}\n"
printf "${GREEN}  ✓ Vloud is up.${RESET}\n"
printf "${GREEN}═══════════════════════════════════════════════════════════════${RESET}\n"
echo
echo "  Server IP:    $PUBLIC_IP"
# Print a URL that actually resolves. The engine binds $VLOUD_BIND_HOST,
# which defaults to 127.0.0.1 (M-04: no reason to expose the upstream
# directly). Advertising http://$PUBLIC_IP:$VLOUD_PORT regardless sent
# operators to an address that refuses every connection — and because
# nginx deliberately does not claim port 80 either, there was no
# working route to the first-run wizard from another machine at all.
if [[ "$VLOUD_BIND_HOST" == "0.0.0.0" || "$VLOUD_BIND_HOST" == "::" ]]; then
  echo "  Dashboard:    http://$PUBLIC_IP:$VLOUD_PORT/onboarding"
else
  echo "  Dashboard:    http://127.0.0.1:$VLOUD_PORT/onboarding   (on this server)"
  echo "                the engine listens on $VLOUD_BIND_HOST only, so the"
  echo "                server's LAN address will refuse the connection."
  echo
  echo "  From another machine, forward the port over SSH:"
  echo "                ssh -L $VLOUD_PORT:127.0.0.1:$VLOUD_PORT <user>@$PUBLIC_IP"
  echo "                then open http://127.0.0.1:$VLOUD_PORT/onboarding"
  echo
  echo "  Or expose it on the network (trusted networks only):"
  echo "                sudo sed -i 's/^VLOUD_BIND_HOST=.*/VLOUD_BIND_HOST=0.0.0.0/' $ENV_FILE"
  echo "                sudo systemctl restart vloud"
fi
echo "  Trial:        ${TRIAL_DAYS_LEFT}-day trial active (state: $TRIAL_STATE)"
echo "  Coexist:      $( [[ "$COEXIST" -eq 1 ]] && echo "yes (cPanel detected)" || echo "no" )"
echo
echo "  Manage:       sudo systemctl {start,stop,restart,status} vloud"
echo "  Tail logs:    sudo journalctl -u vloud -f"
echo "  Env file:     $ENV_FILE  (mode 0640 root:vloud)"
echo "  Install dir:  $VLOUD_INSTALL_DIR"
echo
echo "  Open the dashboard URL above to run the first-run wizard."
echo

exit 0