248 lines
No EOL
8.6 KiB
Bash
248 lines
No EOL
8.6 KiB
Bash
#!/bin/bash
|
|
#
|
|
# Build and deploy dvorakmortonxd.dev.
|
|
#
|
|
# The site is served by PM2 (app name: dvorakmortonxd) from the `deploy` user's
|
|
# PM2 daemon, as described in DEPLOYMENT.md.
|
|
#
|
|
# The build happens in a staging directory and is only swapped into place once
|
|
# it succeeds. That keeps the live site pointed at a complete tree: replacing
|
|
# the directory underneath a running `next start` is what used to make the site
|
|
# serve stale or missing assets.
|
|
#
|
|
# Two things this script is deliberately paranoid about, because both have
|
|
# produced half-finished deploys:
|
|
#
|
|
# * Port ownership. `next start` must own :8080. A leftover process (a manual
|
|
# `npm start`, an old tree, or the copy the legacy root PM2 daemon used to
|
|
# run) makes it die with EADDRINUSE and PM2 just keeps restarting it. We
|
|
# make sure the port is actually free before we bind, and refuse to touch
|
|
# the live deployment while a *foreign* process is sitting on it.
|
|
# * Confirming the *right* server answered. A plain `curl :8080` is happy to
|
|
# talk to whatever is squatting on the port, so a failed deploy looked
|
|
# "online". We now require every listener on :8080 to be a descendant of
|
|
# our own PM2 app, and roll back to the previous build if it is not.
|
|
|
|
set -euo pipefail
|
|
|
|
APP_NAME="dvorakmortonxd"
|
|
APP_DIR="/srv/dvorakmortonxd.dev"
|
|
STAGE_DIR="/srv/dvorakmortonxd.dev.staging"
|
|
OLD_DIR="/srv/dvorakmortonxd.dev.previous"
|
|
FAILED_DIR="/srv/dvorakmortonxd.dev.failed"
|
|
REPO_URL="https://git.dvorakmortonxd.dev/dvorakmortonxd/dvorakmortonxd.dev"
|
|
|
|
# A separate container forwards this host's :8080 to the Cloudflare tunnel, so
|
|
# `next start` has to listen on all interfaces on port 8080 (not localhost:3000).
|
|
PORT="8080"
|
|
BIND_HOST="0.0.0.0"
|
|
HEALTH_URL="http://127.0.0.1:${PORT}/"
|
|
|
|
# How long to wait for the old process to release the port, and for the new one
|
|
# to answer once started.
|
|
PORT_WAIT=15
|
|
HEALTH_WAIT=30
|
|
|
|
log() { echo "==> $*"; }
|
|
warn() { echo "warning: $*" >&2; }
|
|
err() { echo "error: $*" >&2; }
|
|
die() { err "$*"; exit 1; }
|
|
|
|
# Everything runs as `deploy`: it owns /srv and the PM2 daemon in
|
|
# /home/deploy/.pm2. Running the build as another user leaves root-owned files
|
|
# behind and makes git complain about "dubious ownership".
|
|
if [ "$(id -un)" != "deploy" ]; then
|
|
if [ "$(id -u)" -eq 0 ]; then
|
|
exec su - deploy -s /bin/bash -c "$(printf '%q ' "$0" "$@")"
|
|
fi
|
|
echo "error: run this script as the 'deploy' user (or as root)" >&2
|
|
exit 1
|
|
fi
|
|
|
|
# --- port / process helpers -------------------------------------------------
|
|
|
|
# Nothing is listening on $PORT. `ss` reports the socket even when we cannot
|
|
# see the owning process, unlike lsof.
|
|
port_is_free() {
|
|
! ss -H -ltn "sport = :${PORT}" 2>/dev/null | grep -q .
|
|
}
|
|
|
|
# PIDs we can identify as listening on $PORT. Uses ss rather than lsof: this
|
|
# box's lsof does not enumerate IPv6 sockets, and `next start` binds `::`.
|
|
listener_pids() {
|
|
ss -H -ltnp "sport = :${PORT}" 2>/dev/null \
|
|
| grep -oE 'pid=[0-9]+' | cut -d= -f2 | sort -u || true
|
|
}
|
|
|
|
# PID PM2 reports for our app (empty when it is not known to PM2).
|
|
app_pid() {
|
|
pm2 pid "$APP_NAME" 2>/dev/null | tr -d '[:space:]' || true
|
|
}
|
|
|
|
# Exit 0 when $2 is $1 or a descendant of $1.
|
|
is_descendant_of() {
|
|
local ancestor="$1" p="$2"
|
|
while [ -n "$p" ] && [ "$p" -gt 1 ] 2>/dev/null; do
|
|
[ "$p" = "$ancestor" ] && return 0
|
|
p=$(ps -o ppid= -p "$p" 2>/dev/null | tr -d '[:space:]')
|
|
done
|
|
return 1
|
|
}
|
|
|
|
# Exit 0 only when :$PORT is listening *and* every listener is a descendant of
|
|
# our PM2 app. This is the check a bare `curl` cannot make.
|
|
app_owns_port() {
|
|
local pids holder apid
|
|
pids="$(listener_pids)"
|
|
[ -n "$pids" ] || return 1
|
|
apid="$(app_pid)"
|
|
[ -n "$apid" ] && [ "$apid" != "0" ] || return 1
|
|
for holder in $pids; do
|
|
is_descendant_of "$apid" "$holder" || return 1
|
|
done
|
|
return 0
|
|
}
|
|
|
|
# Wait out a normal shutdown, then kill whatever still holds the port.
|
|
free_port() {
|
|
local deadline=$((SECONDS + PORT_WAIT)) pid owner
|
|
while ! port_is_free; do
|
|
[ "$SECONDS" -lt "$deadline" ] || break
|
|
sleep 0.3
|
|
done
|
|
port_is_free && return 0
|
|
|
|
warn "port $PORT is still held; trying to stop the leftover process"
|
|
ss -ltnp "sport = :${PORT}" >&2 2>/dev/null || true
|
|
for pid in $(listener_pids); do
|
|
owner=$(ps -o user= -p "$pid" 2>/dev/null | tr -d '[:space:]')
|
|
if [ "$owner" = "$(id -un)" ]; then
|
|
kill -TERM "$pid" 2>/dev/null || true
|
|
else
|
|
doas kill -TERM "$pid" 2>/dev/null || true
|
|
fi
|
|
done
|
|
sleep 1
|
|
for pid in $(listener_pids); do
|
|
owner=$(ps -o user= -p "$pid" 2>/dev/null | tr -d '[:space:]')
|
|
if [ "$owner" = "$(id -un)" ]; then
|
|
kill -KILL "$pid" 2>/dev/null || true
|
|
else
|
|
doas kill -KILL "$pid" 2>/dev/null || true
|
|
fi
|
|
done
|
|
sleep 1
|
|
port_is_free
|
|
}
|
|
|
|
# Wait for our app to own the port and answer over HTTP.
|
|
wait_for_app() {
|
|
local deadline=$((SECONDS + HEALTH_WAIT))
|
|
while [ "$SECONDS" -lt "$deadline" ]; do
|
|
if app_owns_port && curl -fsS -o /dev/null "$HEALTH_URL"; then
|
|
return 0
|
|
fi
|
|
sleep 1
|
|
done
|
|
return 1
|
|
}
|
|
|
|
# Best-effort removal of the legacy duplicate in root's PM2 daemon. `doas -n`
|
|
# is only allowed to proceed when its `persist` credential is still cached, so
|
|
# this never blocks on a password; the preflight below catches anything left.
|
|
remove_root_duplicate() {
|
|
doas -n env PM2_HOME=/root/.pm2 pm2 delete "$APP_NAME" >/dev/null 2>&1 || true
|
|
doas -n env PM2_HOME=/root/.pm2 pm2 delete website >/dev/null 2>&1 || true
|
|
doas -n env PM2_HOME=/root/.pm2 pm2 save >/dev/null 2>&1 || true
|
|
}
|
|
|
|
# Stop the new (broken) app and bring the previous build back up.
|
|
rollback() {
|
|
err "$APP_NAME did not come up on $HEALTH_URL"
|
|
pm2 logs "$APP_NAME" --lines 40 --nostream >&2 || true
|
|
pm2 delete "$APP_NAME" >/dev/null 2>&1 || true
|
|
free_port >/dev/null 2>&1 || true
|
|
|
|
if [ ! -d "$OLD_DIR" ]; then
|
|
die "no previous build to roll back to"
|
|
fi
|
|
|
|
log "Rolling back to the previous build"
|
|
rm -rf "$FAILED_DIR"
|
|
mv "$APP_DIR" "$FAILED_DIR"
|
|
mv "$OLD_DIR" "$APP_DIR"
|
|
cd "$APP_DIR"
|
|
PORT="$PORT" HOSTNAME="$BIND_HOST" pm2 start npm --name "$APP_NAME" -- start || true
|
|
pm2 save >/dev/null 2>&1 || true
|
|
if wait_for_app; then
|
|
die "deploy failed; rolled back and the previous build is online again"
|
|
fi
|
|
die "deploy failed and the rollback did not come up; check 'pm2 logs $APP_NAME'"
|
|
}
|
|
|
|
# --- build ------------------------------------------------------------------
|
|
|
|
log "Building fresh checkout in $STAGE_DIR"
|
|
rm -rf "$STAGE_DIR" "$FAILED_DIR"
|
|
git clone "$REPO_URL" "$STAGE_DIR"
|
|
|
|
cd "$STAGE_DIR"
|
|
npm ci
|
|
npm run build
|
|
|
|
remove_root_duplicate
|
|
|
|
# Preflight: if :$PORT is already answered by something that is not our app,
|
|
# stop here rather than swapping in a build that can never bind. A legacy root
|
|
# PM2 copy is the usual culprit; try to clear it interactively first.
|
|
if ! port_is_free && ! app_owns_port; then
|
|
if [ -t 0 ]; then
|
|
log "Clearing a conflicting server on :$PORT (legacy root PM2 copy?)"
|
|
doas env PM2_HOME=/root/.pm2 pm2 delete "$APP_NAME" >/dev/null 2>&1 || true
|
|
doas env PM2_HOME=/root/.pm2 pm2 delete website >/dev/null 2>&1 || true
|
|
doas env PM2_HOME=/root/.pm2 pm2 save >/dev/null 2>&1 || true
|
|
sleep 1
|
|
fi
|
|
# Also clear a leftover we own outright (a manual `npm start`, etc.).
|
|
free_port || true
|
|
fi
|
|
if ! port_is_free && ! app_owns_port; then
|
|
err "port $PORT is already in use by a process that is not the '$APP_NAME' PM2 app:"
|
|
ss -ltnp "sport = :${PORT}" >&2 2>/dev/null || true
|
|
err "stop it and re-run. A legacy copy in root's PM2 daemon can be removed with:"
|
|
err " doas env PM2_HOME=/root/.pm2 pm2 delete $APP_NAME"
|
|
die "refusing to deploy on top of an unknown server"
|
|
fi
|
|
|
|
# Stop the running copies *before* swapping the directory: a live next-server
|
|
# keeps reading its files by path, so replacing the tree underneath it is what
|
|
# makes the site serve stale/missing assets. `website` is the legacy app name.
|
|
log "Stopping old process and swapping in the new build"
|
|
pm2 delete "$APP_NAME" >/dev/null 2>&1 || true
|
|
pm2 delete website >/dev/null 2>&1 || true
|
|
|
|
if ! free_port; then
|
|
err "could not free port $PORT; the new build was not installed"
|
|
ss -ltnp "sport = :${PORT}" >&2 2>/dev/null || true
|
|
exit 1
|
|
fi
|
|
|
|
rm -rf "$OLD_DIR"
|
|
if [ -d "$APP_DIR" ]; then
|
|
mv "$APP_DIR" "$OLD_DIR"
|
|
fi
|
|
mv "$STAGE_DIR" "$APP_DIR"
|
|
|
|
cd "$APP_DIR"
|
|
PORT="$PORT" HOSTNAME="$BIND_HOST" pm2 start npm --name "$APP_NAME" -- start
|
|
pm2 save
|
|
|
|
# Fail loudly if the app never actually came up (e.g. the port is still taken),
|
|
# and put the previous build back so a bad deploy cannot take the site down.
|
|
log "Waiting for $APP_NAME to answer on $HEALTH_URL"
|
|
if ! wait_for_app; then
|
|
rollback
|
|
fi
|
|
|
|
rm -rf "$OLD_DIR"
|
|
log "Deployed $(cd "$APP_DIR" && git rev-parse --short HEAD) ($APP_NAME is online)" |