Все проверки выполнены успешно
CI / test (push) Successful in 4m30s
Docker / Build and publish worker image (push) Successful in 17m26s
770 строки
26 KiB
Go
770 строки
26 KiB
Go
package installer
|
|
|
|
import (
|
|
"errors"
|
|
"fmt"
|
|
"os"
|
|
"strings"
|
|
|
|
"rocketgit.ru/rsmon/worker/internal/sshinstall"
|
|
)
|
|
|
|
// ActivationOptions drives work package 4 of docs/source-installation.md:
|
|
// the atomic activation of the staged worker binary. It reuses the
|
|
// classic installer's environment resolution (resolveInstallEnv /
|
|
// renderEnvFile), on-disk layout (resolvePaths), and hardened systemd
|
|
// unit renderer, then installs the binary, env file, data dir, and the
|
|
// detected init's service definition atomically, starts/restarts the
|
|
// worker, and verifies the process and /healthz before declaring the
|
|
// activation a success. Any activation/start/health failure rolls back to
|
|
// the previous working install.
|
|
type ActivationOptions struct {
|
|
// Activate performs the atomic install + service activation after the
|
|
// staging build. When false SourceInstall stops at the staging build
|
|
// (the work-package-3 boundary) and touches no service configuration.
|
|
Activate bool
|
|
|
|
// Name is the instance name ("" = the primary rsmon-worker service).
|
|
// A named instance gets rsmon-worker-<name> paths and its own unit,
|
|
// matching the classic installer's multi-instance layout.
|
|
Name string
|
|
// URL is RSMON_URL (default https://rsmon.ru).
|
|
URL string
|
|
// Token is the worker API token (RSMON_TOKEN). Required for
|
|
// activation; prefer supplying it through a file at the CLI layer so
|
|
// it never appears in the process list.
|
|
Token string
|
|
// PublicURL is the advertised public origin (PUBLIC_URL).
|
|
PublicURL string
|
|
// Host is the worker webapp bind address (WORKER_HOST; default
|
|
// 127.0.0.1 as resolved by the classic installer).
|
|
Host string
|
|
// Port is the worker webapp bind port (WORKER_PORT; default 27401,
|
|
// required for named instances).
|
|
Port string
|
|
// Login / Password are the operator-console basic-auth credentials
|
|
// (WORKER_LOGIN / WORKER_PASSWORD). Both must be set or both empty.
|
|
Login string
|
|
Password string
|
|
// EnvFile is a local systemd-safe env file uploaded instead of the
|
|
// individual knobs. Validated with the classic strict parser.
|
|
EnvFile string
|
|
// NoStart installs the binary, env, data dir, and service definition
|
|
// without starting or restarting the worker.
|
|
NoStart bool
|
|
}
|
|
|
|
// ActivationResult records where and how the staged worker was activated.
|
|
type ActivationResult struct {
|
|
Binary string // installed binary path
|
|
ConfigDir string
|
|
EnvFile string
|
|
DataDir string
|
|
UnitFile string // empty when no service definition was installed
|
|
UnitName string // systemd unit / rc-service name
|
|
Supervisor string // "systemd", "openrc", or "supervisor" (embedded fallback)
|
|
Started bool
|
|
}
|
|
|
|
// installOptions projects the activation knobs onto the classic
|
|
// installer's option shape so the shared env resolution is reused.
|
|
func (a ActivationOptions) installOptions() InstallOptions {
|
|
return InstallOptions{
|
|
URL: a.URL,
|
|
Token: a.Token,
|
|
PublicURL: a.PublicURL,
|
|
Host: a.Host,
|
|
Port: a.Port,
|
|
Login: a.Login,
|
|
Password: a.Password,
|
|
Name: strings.TrimSpace(a.Name),
|
|
NoStart: a.NoStart,
|
|
EnvFile: a.EnvFile,
|
|
}
|
|
}
|
|
|
|
// renderEnv resolves the full worker environment exactly as the classic
|
|
// installer would and renders the canonical systemd-safe env file. The
|
|
// env file (when configured) is read exactly once and validated from the
|
|
// in-memory bytes, so a hostile local writer cannot swap the file
|
|
// between the read used for validation and the read used for the render.
|
|
func (a ActivationOptions) renderEnv() ([]byte, error) {
|
|
name := strings.TrimSpace(a.Name)
|
|
if err := validateInstanceName(name); err != nil {
|
|
return nil, err
|
|
}
|
|
var fileEnv map[string]string
|
|
if a.EnvFile != "" {
|
|
data, err := os.ReadFile(a.EnvFile)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("--env-file: %w", err)
|
|
}
|
|
fe, perr := parseEnvironmentContent(data)
|
|
if perr != nil {
|
|
return nil, fmt.Errorf("--env-file: %w", perr)
|
|
}
|
|
fileEnv = fe
|
|
}
|
|
values, err := resolveInstallEnv(a.installOptions(), name, fileEnv)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return renderEnvFile(values), nil
|
|
}
|
|
|
|
// unitContent renders the service definition for the detected init
|
|
// system. systemd gets the classic hardened unit; OpenRC gets a native
|
|
// init script. An unknown init produces no unit (the no-service gate:
|
|
// the embedded supervisor still installs and manages the worker, but no
|
|
// native service definition is written).
|
|
func unitContent(init sshinstall.InitSystem, p paths) (content string, file string, mode string) {
|
|
svcName := strings.TrimSuffix(p.unitName, ".service")
|
|
switch init {
|
|
case sshinstall.InitSystemd:
|
|
return systemdUnitFor(p), p.unitFile, "0644"
|
|
case sshinstall.InitOpenRC:
|
|
return openrcInitFor(p), "/etc/init.d/" + svcName, "0755"
|
|
default:
|
|
return "", "", ""
|
|
}
|
|
}
|
|
|
|
// runningInitScript reports the init system that is *actually running*
|
|
// (not merely installed): systemd, openrc, or none. The installer writes
|
|
// the service definition for the detected init but drives start/restart
|
|
// through whichever supervisor is usable right now; a container or chroot
|
|
// where no init runs falls back to the embedded supervisor.
|
|
const runningInitScript = `set -eu
|
|
if [ -d /run/systemd/system ] && command -v systemctl >/dev/null 2>&1; then
|
|
echo systemd
|
|
elif [ -e /run/openrc/softlevel ] && command -v rc-service >/dev/null 2>&1; then
|
|
echo openrc
|
|
else
|
|
echo none
|
|
fi`
|
|
|
|
// normalizeSupervisor bounds the running-init probe output to the three
|
|
// supervisor kinds the activation script understands.
|
|
func normalizeSupervisor(raw string) string {
|
|
switch strings.TrimSpace(raw) {
|
|
case "systemd", "openrc":
|
|
return strings.TrimSpace(raw)
|
|
default:
|
|
return "none"
|
|
}
|
|
}
|
|
|
|
// activateScript renders the single idempotent remote activation script.
|
|
// It is structured as a transaction over the previous install:
|
|
//
|
|
// 1. acquire an activation lock so concurrent runs cannot mutate the same
|
|
// deployment, and recover any interrupted prior activation from its
|
|
// leftover backup marker;
|
|
// 2. validate the staged binary (fails before any state is touched);
|
|
// 3. snapshot the prior binary/env/unit (and the unit's enable state)
|
|
// into a backup dir and write a recovery marker;
|
|
// 4. arm a rollback EXIT/HUP/INT/TERM trap that restores the snapshot,
|
|
// restores the prior enable state (or disables a freshly-installed
|
|
// unit on a fresh failure), and restarts the prior service;
|
|
// 5. install the new binary, env (mode 0600), data dir, and service
|
|
// definition atomically (temp file + rename);
|
|
// 6. start/restart through the active supervisor and verify the service
|
|
// stays active (systemd `is-active` / rc-service `status` / the
|
|
// embedded supervisor's zombie-aware pid check) and answers /healthz;
|
|
// 7. on success, drop the backup and release the lock; on failure the
|
|
// trap restores the prior install and the script exits non-zero.
|
|
//
|
|
// Reruns are idempotent: restart always stops the previous instance
|
|
// first, so exactly one worker process exists, and every temp file
|
|
// (backup dir, .new files, uploaded env/unit, lock) is removed on success
|
|
// and failure.
|
|
func activateScript(p activateParams) string {
|
|
r := strings.NewReplacer(
|
|
"@@STAGE@@", shellQuote(p.Stage),
|
|
"@@BINARY@@", shellQuote(p.Binary),
|
|
"@@CONFIG_DIR@@", shellQuote(p.ConfigDir),
|
|
"@@ENV_FILE@@", shellQuote(p.EnvFile),
|
|
"@@ENV_TMP@@", shellQuote(p.EnvTmp),
|
|
"@@DATA_DIR@@", shellQuote(p.DataDir),
|
|
"@@UNIT_TMP@@", shellQuote(p.UnitTmp),
|
|
"@@UNIT_FILE@@", shellQuote(p.UnitFile),
|
|
"@@UNIT_MODE@@", p.UnitMode,
|
|
"@@UNIT_NAME@@", shellQuote(p.UnitName),
|
|
"@@RC_NAME@@", shellQuote(p.RCName),
|
|
"@@SUPERVISOR@@", p.Supervisor,
|
|
"@@NO_START@@", p.NoStart,
|
|
)
|
|
return r.Replace(activateTemplate)
|
|
}
|
|
|
|
// activateTemplate is the remote script body. Sentinels (@@..@@) are
|
|
// substituted by activateScript; every operator-controlled value is
|
|
// single-quoted and validated on the controller side first.
|
|
const activateTemplate = `set -eu
|
|
|
|
stage=@@STAGE@@
|
|
binary=@@BINARY@@
|
|
config_dir=@@CONFIG_DIR@@
|
|
env_file=@@ENV_FILE@@
|
|
env_tmp=@@ENV_TMP@@
|
|
data_dir=@@DATA_DIR@@
|
|
unit_tmp=@@UNIT_TMP@@
|
|
unit_file=@@UNIT_FILE@@
|
|
unit_mode=@@UNIT_MODE@@
|
|
unit_name=@@UNIT_NAME@@
|
|
rc_name=@@RC_NAME@@
|
|
supervisor=@@SUPERVISOR@@
|
|
no_start=@@NO_START@@
|
|
|
|
backup_dir="$data_dir/.rsmon-backup"
|
|
lock_dir="$data_dir/.rsmon-activate.lock"
|
|
pid_file="$data_dir/worker.pid"
|
|
log_file="$data_dir/worker.log"
|
|
|
|
run_env() {
|
|
# Execute "$@" with a fresh environment built only from the env file
|
|
# plus PATH, HOME, and the webapp data dir. Sourcing the env file in a
|
|
# clean env -i shell prevents variables from a previous activation
|
|
# (for example WORKER_CLUSTER_ENABLED) from leaking into the new
|
|
# worker process after a rollback restores an older env file.
|
|
env -i PATH="/usr/bin:/bin:/sbin:/usr/sbin" HOME="$data_dir" \
|
|
RSMON_WEBAPP_DATA_DIR="$data_dir/webapp" \
|
|
RSMON_WORKER_ENV_FILE="$env_file" sh -c '
|
|
set -a
|
|
. "$RSMON_WORKER_ENV_FILE"
|
|
set +a
|
|
unset RSMON_WORKER_ENV_FILE
|
|
exec "$@"
|
|
' sh "$@"
|
|
}
|
|
|
|
process_up() {
|
|
# True when the given pid is a live, non-zombie process. A zombie
|
|
# still answers kill -0 (its task struct exists until reaped), and in
|
|
# a container where PID 1 does not reap children a killed worker can
|
|
# linger as a zombie for a long time, so the /proc state must be
|
|
# checked explicitly.
|
|
[ -n "$1" ] || return 1
|
|
kill -0 "$1" 2>/dev/null || return 1
|
|
state="$(awk '{print $3}' "/proc/$1/stat" 2>/dev/null || true)"
|
|
[ "$state" != "Z" ] || return 1
|
|
}
|
|
|
|
process_is_worker() {
|
|
# True when the given pid's executable is (or was, before an atomic
|
|
# binary swap) the configured worker binary, so a stale or recycled
|
|
# pid file can never make the supervisor kill an unrelated process.
|
|
[ -n "$1" ] || return 1
|
|
exe="$(readlink "/proc/$1/exe" 2>/dev/null || true)"
|
|
case "$exe" in
|
|
"$binary"|"$binary (deleted)") return 0 ;;
|
|
esac
|
|
return 1
|
|
}
|
|
|
|
stop_process() {
|
|
if [ -f "$pid_file" ]; then
|
|
pid="$(cat "$pid_file" 2>/dev/null || true)"
|
|
if [ -n "$pid" ] && process_is_worker "$pid" && process_up "$pid"; then
|
|
kill "$pid" 2>/dev/null || true
|
|
i=0
|
|
while [ "$i" -lt 10 ]; do
|
|
if ! process_up "$pid"; then
|
|
break
|
|
fi
|
|
sleep 1
|
|
i=$((i + 1))
|
|
done
|
|
kill -9 "$pid" 2>/dev/null || true
|
|
fi
|
|
fi
|
|
rm -f "$pid_file"
|
|
}
|
|
|
|
start_process() {
|
|
stop_process
|
|
mkdir -p "$data_dir"
|
|
run_env sh -c 'nohup "$1" >>"$2" 2>&1 & echo $! > "$3"' sh "$binary" "$log_file" "$pid_file"
|
|
}
|
|
|
|
process_alive() {
|
|
[ -f "$pid_file" ] || return 1
|
|
pid="$(cat "$pid_file" 2>/dev/null || true)"
|
|
process_is_worker "$pid" || return 1
|
|
process_up "$pid"
|
|
}
|
|
|
|
worker_come_up() {
|
|
# Wait up to 10s for the freshly-started worker to become a live,
|
|
# non-zombie process running the configured binary. The recorded pid
|
|
# starts as the nohup/sh child before it execs the worker, so early
|
|
# readlink checks can transiently see the interpreter; this retry both
|
|
# tolerates that exec window and catches genuine immediate deaths.
|
|
i=0
|
|
while [ "$i" -lt 10 ]; do
|
|
if process_alive; then
|
|
return 0
|
|
fi
|
|
sleep 1
|
|
i=$((i + 1))
|
|
done
|
|
return 1
|
|
}
|
|
|
|
health_ok() {
|
|
run_env "$binary" liveness >/dev/null 2>&1
|
|
}
|
|
|
|
svc_active() {
|
|
# The health loop verifies the service is active through the
|
|
# supervisor (never through a pid file for the init-managed paths).
|
|
case "$supervisor" in
|
|
systemd) systemctl is-active --quiet "$unit_name" ;;
|
|
openrc) rc-service "$rc_name" status >/dev/null 2>&1 ;;
|
|
none) process_alive ;;
|
|
esac
|
|
}
|
|
|
|
stop_svc() {
|
|
case "$supervisor" in
|
|
systemd) systemctl stop "$unit_name" >/dev/null 2>&1 || true ;;
|
|
openrc) rc-service "$rc_name" stop >/dev/null 2>&1 || true ;;
|
|
none) stop_process ;;
|
|
esac
|
|
}
|
|
|
|
restart_svc() {
|
|
case "$supervisor" in
|
|
systemd)
|
|
systemctl daemon-reload >/dev/null 2>&1
|
|
if ! systemctl restart "$unit_name" >/dev/null 2>&1; then
|
|
echo "start failure: systemctl restart $unit_name failed" >&2
|
|
return 1
|
|
fi
|
|
if ! systemctl is-active --quiet "$unit_name"; then
|
|
echo "start failure: unit $unit_name is not active" >&2
|
|
return 1
|
|
fi
|
|
;;
|
|
openrc)
|
|
if ! rc-service "$rc_name" restart >/dev/null 2>&1; then
|
|
echo "start failure: rc-service restart $rc_name failed" >&2
|
|
return 1
|
|
fi
|
|
if ! rc-service "$rc_name" status >/dev/null 2>&1; then
|
|
echo "start failure: service $rc_name is not running" >&2
|
|
return 1
|
|
fi
|
|
;;
|
|
none)
|
|
start_process
|
|
if ! worker_come_up; then
|
|
echo "start failure: worker process did not stay alive" >&2
|
|
return 1
|
|
fi
|
|
;;
|
|
esac
|
|
i=0
|
|
while [ "$i" -lt 30 ]; do
|
|
if health_ok; then
|
|
return 0
|
|
fi
|
|
# A worker that stops while the health gate is still probing is a
|
|
# start failure (it never became a stable service), not merely an
|
|
# unhealthy-but-running one.
|
|
if ! svc_active; then
|
|
echo "start failure: worker service is not active" >&2
|
|
return 1
|
|
fi
|
|
sleep 1
|
|
i=$((i + 1))
|
|
done
|
|
echo "health failure: worker did not answer /healthz" >&2
|
|
return 1
|
|
}
|
|
|
|
# --- unit enable helpers -------------------------------------------------
|
|
# unit_was_enabled records whether the unit referenced by the current
|
|
# params is enabled (best-effort; a missing init leaves it disabled).
|
|
unit_was_enabled() {
|
|
case "$supervisor" in
|
|
systemd) systemctl is-enabled "$unit_name" >/dev/null 2>&1 && unit_enabled=1 ;;
|
|
openrc) [ -e "/etc/runlevels/default/$rc_name" ] && unit_enabled=1 ;;
|
|
esac
|
|
return 0
|
|
}
|
|
# set_unit_enabled "$1" enables (1) or disables (0) the unit.
|
|
set_unit_enabled() {
|
|
case "$supervisor" in
|
|
systemd)
|
|
systemctl daemon-reload >/dev/null 2>&1 || true
|
|
if [ "$1" -eq 1 ]; then
|
|
systemctl enable "$unit_name" >/dev/null 2>&1 || true
|
|
else
|
|
systemctl disable "$unit_name" >/dev/null 2>&1 || true
|
|
fi
|
|
;;
|
|
openrc)
|
|
if [ "$1" -eq 1 ]; then
|
|
rc-update add "$rc_name" default >/dev/null 2>&1 || true
|
|
else
|
|
rc-update del "$rc_name" default >/dev/null 2>&1 || true
|
|
fi
|
|
;;
|
|
esac
|
|
return 0
|
|
}
|
|
|
|
# --- activation lock ------------------------------------------------------
|
|
# Prevent concurrent activations on the same host from racing the
|
|
# deployed-state mutation. A stale lock (left over from a SIGKILL'd run
|
|
# whose pid is no longer alive) is broken automatically.
|
|
acquire_lock() {
|
|
mkdir -p "$data_dir"
|
|
if [ -d "$lock_dir" ]; then
|
|
lockpid="$(cat "$lock_dir/pid" 2>/dev/null || true)"
|
|
if [ -n "$lockpid" ] && ! kill -0 "$lockpid" 2>/dev/null; then
|
|
rm -rf "$lock_dir" 2>/dev/null || true
|
|
fi
|
|
fi
|
|
i=0
|
|
while [ "$i" -lt 60 ]; do
|
|
if mkdir "$lock_dir" 2>/dev/null; then
|
|
chmod 0700 "$lock_dir" 2>/dev/null || true
|
|
echo $$ > "$lock_dir/pid" 2>/dev/null || true
|
|
return 0
|
|
fi
|
|
sleep 1
|
|
i=$((i + 1))
|
|
done
|
|
echo "another rsmon-worker activation is in progress ($lock_dir)" >&2
|
|
return 1
|
|
}
|
|
release_lock() {
|
|
rm -rf "$lock_dir" 2>/dev/null || true
|
|
}
|
|
|
|
if ! acquire_lock; then
|
|
exit 1
|
|
fi
|
|
# Any early failure (before the rollback trap is armed) still releases the
|
|
# lock.
|
|
trap 'release_lock' EXIT
|
|
|
|
# --- interrupted-run recovery --------------------------------------------
|
|
# A previous activation that was killed mid-flight (SSH drop, SIGKILL)
|
|
# leaves its backup marker behind. Restore that snapshot before the fresh
|
|
# activation runs, so the host never stays half-activated and the next run
|
|
# starts from a consistent prior state.
|
|
unit_enabled=0
|
|
recover_backup() {
|
|
if [ ! -f "$backup_dir/.marker" ]; then
|
|
return 0
|
|
fi
|
|
echo "recovering interrupted activation from $backup_dir" >&2
|
|
if [ -f "$backup_dir/binary" ]; then
|
|
mkdir -p "$(dirname "$binary")" || true
|
|
cp -p "$backup_dir/binary" "$binary" || true
|
|
fi
|
|
if [ -f "$backup_dir/worker.env" ]; then
|
|
mkdir -p "$config_dir" || true
|
|
cp -p "$backup_dir/worker.env" "$env_file" || true
|
|
fi
|
|
if [ -n "$unit_file" ] && [ -f "$backup_dir/unit" ]; then
|
|
mkdir -p "$(dirname "$unit_file")" || true
|
|
cp -p "$backup_dir/unit" "$unit_file" || true
|
|
chmod "$unit_mode" "$unit_file" || true
|
|
fi
|
|
if [ "$(cat "$backup_dir/.marker" 2>/dev/null || true)" = "1" ]; then
|
|
unit_enabled=1
|
|
fi
|
|
set_unit_enabled "$unit_enabled" || true
|
|
if [ -f "$backup_dir/binary" ] && [ -f "$backup_dir/worker.env" ] && [ "$no_start" -ne 1 ]; then
|
|
restart_svc || true
|
|
fi
|
|
rm -rf "$backup_dir" || true
|
|
}
|
|
recover_backup
|
|
|
|
# --- staged binary validation (no state touched) -------------------------
|
|
# A corrupt or missing staging build aborts here and leaves the prior
|
|
# install completely untouched (the rollback trap is not armed yet).
|
|
"$stage" --version >/dev/null
|
|
|
|
# --- snapshot the prior install ------------------------------------------
|
|
rm -rf "$backup_dir"
|
|
mkdir -p "$backup_dir"
|
|
had_binary=0
|
|
had_env=0
|
|
had_unit=0
|
|
unit_enabled=0
|
|
if [ -e "$binary" ]; then
|
|
cp -p "$binary" "$backup_dir/binary"
|
|
had_binary=1
|
|
fi
|
|
if [ -e "$env_file" ]; then
|
|
cp -p "$env_file" "$backup_dir/worker.env"
|
|
had_env=1
|
|
fi
|
|
if [ -n "$unit_file" ] && [ -e "$unit_file" ]; then
|
|
had_unit=1
|
|
cp -p "$unit_file" "$backup_dir/unit"
|
|
unit_was_enabled
|
|
fi
|
|
# The marker is written only after the snapshot copies so recovery never
|
|
# sees a partial snapshot; it records whether the prior unit was enabled.
|
|
printf '%s\n' "$unit_enabled" > "$backup_dir/.marker"
|
|
|
|
# --- rollback trap --------------------------------------------------------
|
|
# Any failure from here on restores the snapshot (preserving binary/env/unit
|
|
# metadata), restores the prior enable state (or disables a freshly-installed
|
|
# unit on a fresh install), and brings the prior service back up. The trap
|
|
# also fires on HUP/INT/TERM so an interrupted run rolls back instead of
|
|
# leaving a half-activated state.
|
|
rolled_back=0
|
|
rollback() {
|
|
[ "$rolled_back" -eq 1 ] && return 0
|
|
rolled_back=1
|
|
echo "rsmon-worker activation failed; restoring the prior install" >&2
|
|
stop_svc || true
|
|
rm -f "$binary.new" "$env_file.new" "$unit_file.new" "$pid_file" || true
|
|
if [ "$had_binary" -eq 1 ]; then
|
|
cp -p "$backup_dir/binary" "$binary" || true
|
|
elif [ -e "$binary" ]; then
|
|
rm -f "$binary" || true
|
|
fi
|
|
if [ "$had_env" -eq 1 ]; then
|
|
cp -p "$backup_dir/worker.env" "$env_file" || true
|
|
chmod 0600 "$env_file" || true
|
|
elif [ -e "$env_file" ]; then
|
|
rm -f "$env_file" || true
|
|
fi
|
|
if [ -n "$unit_file" ]; then
|
|
if [ "$had_unit" -eq 1 ]; then
|
|
cp -p "$backup_dir/unit" "$unit_file" || true
|
|
chmod "$unit_mode" "$unit_file" || true
|
|
set_unit_enabled "$unit_enabled" || true
|
|
else
|
|
set_unit_enabled 0 || true
|
|
rm -f "$unit_file" || true
|
|
fi
|
|
fi
|
|
rm -f "$env_tmp" "$unit_tmp" || true
|
|
rm -rf "$backup_dir" || true
|
|
if [ "$had_binary" -eq 1 ] && [ "$had_env" -eq 1 ] && [ "$no_start" -ne 1 ]; then
|
|
restart_svc || true
|
|
fi
|
|
release_lock
|
|
exit 1
|
|
}
|
|
trap rollback EXIT HUP INT TERM
|
|
|
|
# --- atomic binary install ------------------------------------------------
|
|
mkdir -p "$(dirname "$binary")"
|
|
install -m 0755 "$stage" "$binary.new"
|
|
mv -f "$binary.new" "$binary"
|
|
|
|
# --- atomic env install (secrets, mode 0600) ------------------------------
|
|
mkdir -p "$config_dir"
|
|
chmod 0750 "$config_dir"
|
|
install -m 0600 "$env_tmp" "$env_file.new"
|
|
mv -f "$env_file.new" "$env_file"
|
|
rm -f "$env_tmp"
|
|
|
|
# --- data directory -------------------------------------------------------
|
|
mkdir -p "$data_dir" "$data_dir/webapp"
|
|
chmod 0755 "$data_dir" "$data_dir/webapp"
|
|
|
|
# --- service definition ---------------------------------------------------
|
|
if [ -n "$unit_file" ] && [ -f "$unit_tmp" ]; then
|
|
mkdir -p "$(dirname "$unit_file")"
|
|
install -m "$unit_mode" "$unit_tmp" "$unit_file.new"
|
|
mv -f "$unit_file.new" "$unit_file"
|
|
rm -f "$unit_tmp"
|
|
set_unit_enabled 1
|
|
fi
|
|
|
|
# --- start and verify -----------------------------------------------------
|
|
# A failed start or a worker that does not answer /healthz triggers the
|
|
# rollback trap.
|
|
if [ "$no_start" -ne 1 ]; then
|
|
if ! restart_svc; then
|
|
echo "rsmon-worker activation failed: process or /healthz verification failed" >&2
|
|
exit 1
|
|
fi
|
|
else
|
|
stop_svc || true
|
|
fi
|
|
|
|
# --- success --------------------------------------------------------------
|
|
rm -rf "$backup_dir"
|
|
rm -f "$env_tmp" "$unit_tmp"
|
|
release_lock
|
|
trap - EXIT HUP INT TERM
|
|
if [ "$no_start" -eq 1 ]; then
|
|
echo "rsmon-worker activated: binary=$binary supervisor=$supervisor unit=${unit_file:-none} started=no"
|
|
else
|
|
echo "rsmon-worker activated: binary=$binary supervisor=$supervisor unit=${unit_file:-none} started=yes"
|
|
fi
|
|
exit 0
|
|
`
|
|
|
|
// activateParams are the resolved, controller-validated values fed into
|
|
// the remote activation script.
|
|
type activateParams struct {
|
|
Stage string
|
|
Binary string
|
|
ConfigDir string
|
|
EnvFile string
|
|
EnvTmp string
|
|
DataDir string
|
|
UnitTmp string
|
|
UnitFile string
|
|
UnitMode string
|
|
UnitName string // systemd unit name (rsmon-worker.service)
|
|
RCName string // rc-service name (rsmon-worker)
|
|
Supervisor string // systemd | openrc | none
|
|
NoStart string // 0 or 1
|
|
}
|
|
|
|
// activateWorker performs the work-package-4 activation for an already
|
|
// staged source build over the live SSH executor. It uses the env bytes
|
|
// rendered once during option normalization, uploads the env and service
|
|
// definition into a server-created 0700 temp dir (no /tmp symlink/TOCTOU
|
|
// attack surface), runs the atomic activation script, and records where
|
|
// and how the worker was activated. Every remote temp file it creates is
|
|
// removed on success and failure.
|
|
func (e *sourceExecutor) activateWorker(opts SourceInstallOptions, res *SourceInstallResult) error {
|
|
a := opts.Activation
|
|
name := strings.TrimSpace(a.Name)
|
|
p := resolvePaths(name)
|
|
|
|
envData := opts.activationEnv
|
|
if len(envData) == 0 {
|
|
return errors.New("activation environment was not rendered (internal error)")
|
|
}
|
|
|
|
unitContent, unitFile, unitMode := unitContent(res.Detection.InitSystem, p)
|
|
svcName := strings.TrimSuffix(p.unitName, ".service")
|
|
|
|
supOut, err := e.runPlain("sh -c " + shellQuote(runningInitScript))
|
|
if err != nil {
|
|
return fmt.Errorf("detect running init system: %w", err)
|
|
}
|
|
supervisor := normalizeSupervisor(string(supOut))
|
|
|
|
// Create a server-side 0700 temp dir owned by the SSH user, then
|
|
// upload the env/unit into it. mktemp -d produces an unpredictable,
|
|
// private path, so a hostile local user cannot pre-create a symlink at
|
|
// a predictable /tmp name (the classic upload TOCTOU). The token
|
|
// never appears in argv or logs: it travels only as base64 over the
|
|
// session stdin and later only inside the mode-0600 env file.
|
|
out, err := e.runPlain("d=$(mktemp -d /tmp/rsmon-worker-act.XXXXXX) && chmod 0700 \"$d\" && echo \"$d\"")
|
|
if err != nil {
|
|
return fmt.Errorf("create secure upload directory: %w", err)
|
|
}
|
|
secureDir := strings.TrimSpace(string(out))
|
|
if secureDir == "" || !strings.HasPrefix(secureDir, "/tmp/") {
|
|
return errors.New("remote returned an invalid secure upload directory")
|
|
}
|
|
envTmp := secureDir + "/worker.env"
|
|
unitTmp := secureDir + "/unit"
|
|
if err := uploadBytes(e.client, envData, envTmp, 0o600); err != nil {
|
|
return fmt.Errorf("upload worker environment: %w", err)
|
|
}
|
|
if unitContent != "" {
|
|
if err := uploadBytes(e.client, []byte(unitContent), unitTmp, 0o644); err != nil {
|
|
return fmt.Errorf("upload service definition: %w", err)
|
|
}
|
|
}
|
|
defer func() {
|
|
_ = runRemote(e.client, "rm -rf -- "+shellQuote(secureDir), nil)
|
|
}()
|
|
|
|
noStart := "0"
|
|
if a.NoStart {
|
|
noStart = "1"
|
|
}
|
|
script := activateScript(activateParams{
|
|
Stage: res.StageBinary,
|
|
Binary: p.binary,
|
|
ConfigDir: p.configDir,
|
|
EnvFile: p.envFile,
|
|
EnvTmp: envTmp,
|
|
DataDir: p.dataDir,
|
|
UnitTmp: unitTmp,
|
|
UnitFile: unitFile,
|
|
UnitMode: unitMode,
|
|
UnitName: p.unitName,
|
|
RCName: svcName,
|
|
Supervisor: supervisor,
|
|
NoStart: noStart,
|
|
})
|
|
if _, err := e.runPrivileged("sh -c " + shellQuote(script)); err != nil {
|
|
return fmt.Errorf("activate worker service: %w", err)
|
|
}
|
|
|
|
res.Activation = &ActivationResult{
|
|
Binary: p.binary,
|
|
ConfigDir: p.configDir,
|
|
EnvFile: p.envFile,
|
|
DataDir: p.dataDir,
|
|
UnitFile: unitFile,
|
|
UnitName: p.unitName,
|
|
Supervisor: supervisor,
|
|
Started: !a.NoStart,
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// validateRenderedEnv runs the strict environment-file parser over the
|
|
// rendered env bytes in memory so the exact bytes written remotely pass
|
|
// the same gate as a user-supplied file, without re-reading a file.
|
|
func validateRenderedEnv(data []byte) error {
|
|
_, err := parseEnvironmentContent(data)
|
|
return err
|
|
}
|
|
|
|
// openrcInitFor renders the native OpenRC init script for an instance. It
|
|
// mirrors the hardening of the systemd unit (data dir, env file, webapp
|
|
// data dir) using OpenRC conventions; it is written for real hosts where
|
|
// openrc is the running init, while containers that cannot run openrc
|
|
// fall back to the embedded supervisor.
|
|
func openrcInitFor(p paths) string {
|
|
description := "RSMon distributed monitoring worker"
|
|
svcName := strings.TrimSuffix(p.unitName, ".service")
|
|
if p.name != "" {
|
|
description += " (" + p.name + ")"
|
|
}
|
|
return fmt.Sprintf(`#!/sbin/openrc-run
|
|
# Managed by the rsmon-worker source installer; do not edit by hand.
|
|
|
|
name=%s
|
|
description=%s
|
|
command=%s
|
|
command_background=true
|
|
pidfile=%s/worker.pid
|
|
output_log=%s/worker.log
|
|
error_log=%s/worker.log
|
|
|
|
depend() {
|
|
need net
|
|
}
|
|
|
|
start_pre() {
|
|
checkpath --directory --mode 0755 --owner root:root %s %s/webapp
|
|
if [ -f %s ]; then
|
|
# OpenRC runs the command with the init script's environment, so
|
|
# the env-file variables must be exported (set -a) or the worker
|
|
# would start without them.
|
|
set -a
|
|
. %s
|
|
set +a
|
|
fi
|
|
export HOME=%s
|
|
export RSMON_WEBAPP_DATA_DIR=%s/webapp
|
|
}
|
|
`, svcName, description, p.binary, p.dataDir, p.dataDir, p.dataDir,
|
|
p.dataDir, p.dataDir, p.envFile, p.envFile, p.dataDir, p.dataDir)
|
|
}
|