#!/bin/bash # # Build and deploy dvorakmortonxd.dev. # # The site is served by PM2 (app name: dvorakmortonxd) from the `deploy` user's # PM2 daemon, as described in DEPLOYMENT.md. # # The build happens in a staging directory and is only swapped into place once # it succeeds. That keeps the live site pointed at a complete tree: replacing # the directory underneath a running `next start` is what used to make the site # serve stale or missing assets. # # Two things this script is deliberately paranoid about, because both have # produced half-finished deploys: # # * Port ownership. `next start` must own :8080. A leftover process (a manual # `npm start`, an old tree, or the copy the legacy root PM2 daemon used to # run) makes it die with EADDRINUSE and PM2 just keeps restarting it. We # make sure the port is actually free before we bind, and refuse to touch # the live deployment while a *foreign* process is sitting on it. # * Confirming the *right* server answered. A plain `curl :8080` is happy to # talk to whatever is squatting on the port, so a failed deploy looked # "online". We now require every listener on :8080 to be a descendant of # our own PM2 app, and roll back to the previous build if it is not. set -euo pipefail APP_NAME="dvorakmortonxd" APP_DIR="/srv/dvorakmortonxd.dev" STAGE_DIR="/srv/dvorakmortonxd.dev.staging" OLD_DIR="/srv/dvorakmortonxd.dev.previous" FAILED_DIR="/srv/dvorakmortonxd.dev.failed" REPO_URL="https://git.dvorakmortonxd.dev/dvorakmortonxd/dvorakmortonxd.dev" # A separate container forwards this host's :8080 to the Cloudflare tunnel, so # `next start` has to listen on all interfaces on port 8080 (not localhost:3000). PORT="8080" BIND_HOST="0.0.0.0" HEALTH_URL="http://127.0.0.1:${PORT}/" # How long to wait for the old process to release the port, and for the new one # to answer once started. PORT_WAIT=15 HEALTH_WAIT=30 log() { echo "==> $*"; } warn() { echo "warning: $*" >&2; } err() { echo "error: $*" >&2; } die() { err "$*"; exit 1; } # Everything runs as `deploy`: it owns /srv and the PM2 daemon in # /home/deploy/.pm2. Running the build as another user leaves root-owned files # behind and makes git complain about "dubious ownership". if [ "$(id -un)" != "deploy" ]; then if [ "$(id -u)" -eq 0 ]; then exec su - deploy -s /bin/bash -c "$(printf '%q ' "$0" "$@")" fi echo "error: run this script as the 'deploy' user (or as root)" >&2 exit 1 fi # --- port / process helpers ------------------------------------------------- # Nothing is listening on $PORT. `ss` reports the socket even when we cannot # see the owning process, unlike lsof. port_is_free() { ! ss -H -ltn "sport = :${PORT}" 2>/dev/null | grep -q . } # PIDs we can identify as listening on $PORT. Uses ss rather than lsof: this # box's lsof does not enumerate IPv6 sockets, and `next start` binds `::`. listener_pids() { ss -H -ltnp "sport = :${PORT}" 2>/dev/null \ | grep -oE 'pid=[0-9]+' | cut -d= -f2 | sort -u || true } # PID PM2 reports for our app (empty when it is not known to PM2). app_pid() { pm2 pid "$APP_NAME" 2>/dev/null | tr -d '[:space:]' || true } # Exit 0 when $2 is $1 or a descendant of $1. is_descendant_of() { local ancestor="$1" p="$2" while [ -n "$p" ] && [ "$p" -gt 1 ] 2>/dev/null; do [ "$p" = "$ancestor" ] && return 0 p=$(ps -o ppid= -p "$p" 2>/dev/null | tr -d '[:space:]') done return 1 } # Exit 0 only when :$PORT is listening *and* every listener is a descendant of # our PM2 app. This is the check a bare `curl` cannot make. app_owns_port() { local pids holder apid pids="$(listener_pids)" [ -n "$pids" ] || return 1 apid="$(app_pid)" [ -n "$apid" ] && [ "$apid" != "0" ] || return 1 for holder in $pids; do is_descendant_of "$apid" "$holder" || return 1 done return 0 } # Wait out a normal shutdown, then kill whatever still holds the port. free_port() { local deadline=$((SECONDS + PORT_WAIT)) pid owner while ! port_is_free; do [ "$SECONDS" -lt "$deadline" ] || break sleep 0.3 done port_is_free && return 0 warn "port $PORT is still held; trying to stop the leftover process" ss -ltnp "sport = :${PORT}" >&2 2>/dev/null || true for pid in $(listener_pids); do owner=$(ps -o user= -p "$pid" 2>/dev/null | tr -d '[:space:]') if [ "$owner" = "$(id -un)" ]; then kill -TERM "$pid" 2>/dev/null || true else doas kill -TERM "$pid" 2>/dev/null || true fi done sleep 1 for pid in $(listener_pids); do owner=$(ps -o user= -p "$pid" 2>/dev/null | tr -d '[:space:]') if [ "$owner" = "$(id -un)" ]; then kill -KILL "$pid" 2>/dev/null || true else doas kill -KILL "$pid" 2>/dev/null || true fi done sleep 1 port_is_free } # Wait for our app to own the port and answer over HTTP. wait_for_app() { local deadline=$((SECONDS + HEALTH_WAIT)) while [ "$SECONDS" -lt "$deadline" ]; do if app_owns_port && curl -fsS -o /dev/null "$HEALTH_URL"; then return 0 fi sleep 1 done return 1 } # Best-effort removal of the legacy duplicate in root's PM2 daemon. `doas -n` # is only allowed to proceed when its `persist` credential is still cached, so # this never blocks on a password; the preflight below catches anything left. remove_root_duplicate() { doas -n env PM2_HOME=/root/.pm2 pm2 delete "$APP_NAME" >/dev/null 2>&1 || true doas -n env PM2_HOME=/root/.pm2 pm2 delete website >/dev/null 2>&1 || true doas -n env PM2_HOME=/root/.pm2 pm2 save >/dev/null 2>&1 || true } # Stop the new (broken) app and bring the previous build back up. rollback() { err "$APP_NAME did not come up on $HEALTH_URL" pm2 logs "$APP_NAME" --lines 40 --nostream >&2 || true pm2 delete "$APP_NAME" >/dev/null 2>&1 || true free_port >/dev/null 2>&1 || true if [ ! -d "$OLD_DIR" ]; then die "no previous build to roll back to" fi log "Rolling back to the previous build" rm -rf "$FAILED_DIR" mv "$APP_DIR" "$FAILED_DIR" mv "$OLD_DIR" "$APP_DIR" cd "$APP_DIR" PORT="$PORT" HOSTNAME="$BIND_HOST" pm2 start npm --name "$APP_NAME" -- start || true pm2 save >/dev/null 2>&1 || true if wait_for_app; then die "deploy failed; rolled back and the previous build is online again" fi die "deploy failed and the rollback did not come up; check 'pm2 logs $APP_NAME'" } # --- build ------------------------------------------------------------------ log "Building fresh checkout in $STAGE_DIR" rm -rf "$STAGE_DIR" "$FAILED_DIR" git clone "$REPO_URL" "$STAGE_DIR" cd "$STAGE_DIR" npm ci npm run build remove_root_duplicate # Preflight: if :$PORT is already answered by something that is not our app, # stop here rather than swapping in a build that can never bind. A legacy root # PM2 copy is the usual culprit; try to clear it interactively first. if ! port_is_free && ! app_owns_port; then if [ -t 0 ]; then log "Clearing a conflicting server on :$PORT (legacy root PM2 copy?)" doas env PM2_HOME=/root/.pm2 pm2 delete "$APP_NAME" >/dev/null 2>&1 || true doas env PM2_HOME=/root/.pm2 pm2 delete website >/dev/null 2>&1 || true doas env PM2_HOME=/root/.pm2 pm2 save >/dev/null 2>&1 || true sleep 1 fi # Also clear a leftover we own outright (a manual `npm start`, etc.). free_port || true fi if ! port_is_free && ! app_owns_port; then err "port $PORT is already in use by a process that is not the '$APP_NAME' PM2 app:" ss -ltnp "sport = :${PORT}" >&2 2>/dev/null || true err "stop it and re-run. A legacy copy in root's PM2 daemon can be removed with:" err " doas env PM2_HOME=/root/.pm2 pm2 delete $APP_NAME" die "refusing to deploy on top of an unknown server" fi # Stop the running copies *before* swapping the directory: a live next-server # keeps reading its files by path, so replacing the tree underneath it is what # makes the site serve stale/missing assets. `website` is the legacy app name. log "Stopping old process and swapping in the new build" pm2 delete "$APP_NAME" >/dev/null 2>&1 || true pm2 delete website >/dev/null 2>&1 || true if ! free_port; then err "could not free port $PORT; the new build was not installed" ss -ltnp "sport = :${PORT}" >&2 2>/dev/null || true exit 1 fi rm -rf "$OLD_DIR" if [ -d "$APP_DIR" ]; then mv "$APP_DIR" "$OLD_DIR" fi mv "$STAGE_DIR" "$APP_DIR" cd "$APP_DIR" PORT="$PORT" HOSTNAME="$BIND_HOST" pm2 start npm --name "$APP_NAME" -- start pm2 save # Fail loudly if the app never actually came up (e.g. the port is still taken), # and put the previous build back so a bad deploy cannot take the site down. log "Waiting for $APP_NAME to answer on $HEALTH_URL" if ! wait_for_app; then rollback fi rm -rf "$OLD_DIR" log "Deployed $(cd "$APP_DIR" && git rev-parse --short HEAD) ($APP_NAME is online)"