feat(installer): activate source builds atomically

internal/installer/sourceactivate_shell_test.go

@@ -0,0 +1,438 @@
+package installer
+
+import (
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"testing"
+)
+
+// This file exercises the rendered activation script against stub
+// systemd/OpenRC implementations ("targeted shell fixture tests"). Real
+// systemd and OpenRC cannot run inside the Docker/OpenSSH harness (the
+// container PID 1 is sshd), so these tests execute the actual remote
+// script on the host with fake `systemctl`, `rc-service`, and `rc-update`
+// binaries that simulate an init-managed host. This covers the unit
+// install/enable, the supervisor-driven restart, the `is-active`-based
+// health loop, the enable-state rollback, the activation lock, and the
+// interrupted-run recovery paths that the E2E fixtures cannot reach.
+//
+// Residual limitation: the stub init tools verify command flow and state
+// transitions, not the real systemd/OpenRC unit semantics; a real
+// init-system smoke test remains out of scope for the container harness.
+
+// systemctlStub records every invocation and simulates unit state in a
+// fake state directory: `is-active`/`is-enabled` consult marker files,
+// `restart` marks the unit active (or fails when the fail-restart marker
+// is present), and `enable`/`disable` toggle the enabled marker.
+const systemctlStub = `#!/bin/sh
+echo "$*" >> "$SYSCTL_LOG"
+action="$1"
+shift
+unit=""
+for a in "$@"; do
+    case "$a" in --*) continue ;; esac
+    unit="$a"
+    break
+done
+case "$action" in
+    is-active)  [ -f "$FAKE_SYSTEMD_DIR/$unit.active" ] || exit 3 ;;
+    is-enabled) [ -f "$FAKE_SYSTEMD_DIR/$unit.enabled" ] || exit 1 ;;
+    enable)     touch "$FAKE_SYSTEMD_DIR/$unit.enabled"; exit 0 ;;
+    disable)    rm -f "$FAKE_SYSTEMD_DIR/$unit.enabled"; exit 0 ;;
+    daemon-reload) exit 0 ;;
+    restart)    [ -f "$FAKE_SYSTEMD_DIR/fail-restart" ] && exit 1
+                touch "$FAKE_SYSTEMD_DIR/$unit.active"; exit 0 ;;
+    stop)       rm -f "$FAKE_SYSTEMD_DIR/$unit.active"; exit 0 ;;
+    *) exit 0 ;;
+esac
+`
+
+// rcServiceStub simulates OpenRC service state; rcUpdateStub simulates
+// runlevel enablement.
+const rcServiceStub = `#!/bin/sh
+echo "$*" >> "$RC_LOG"
+svc="$1"
+action="$2"
+case "$action" in
+    restart) [ -f "$FAKE_RC_DIR/fail-restart" ] && exit 1
+             touch "$FAKE_RC_DIR/$svc.active"; exit 0 ;;
+    status)  [ -f "$FAKE_RC_DIR/$svc.active" ] || exit 1 ;;
+    stop)    rm -f "$FAKE_RC_DIR/$svc.active"; exit 0 ;;
+    *) exit 0 ;;
+esac
+`
+
+const rcUpdateStub = `#!/bin/sh
+echo "$*" >> "$RCU_LOG"
+case "$1" in
+    add) touch "$FAKE_RC_DIR/$2.enabled"; exit 0 ;;
+    del) rm -f "$FAKE_RC_DIR/$2.enabled"; exit 0 ;;
+    *) exit 0 ;;
+esac
+`
+
+// workerStub stands in for the built worker binary: it satisfies the
+// `--version` validation and the `/healthz` liveness probe.
+const workerStub = `#!/bin/sh
+if [ "$1" = "--version" ]; then
+    echo "rsmon-worker version=dev commit=deadbeefdead buildDate=2026-01-01T00:00:00Z"
+    exit 0
+fi
+if [ "$1" = "liveness" ]; then
+    exit 0
+fi
+exit 1
+`
+
+// initShellFixture builds a temp "host" with stub init tools and a stub
+// worker, plus the uploaded env/unit the activation script consumes.
+type initShellFixture struct {
+	root    string
+	binDir  string
+	sysdDir string
+	rcDir   string
+
+	sysctlLog string
+	rcLog     string
+	rcuLog    string
+
+	params activateParams
+}
+
+func writeExec(t *testing.T, path, content string) {
+	t.Helper()
+	if err := os.WriteFile(path, []byte(content), 0o700); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func newInitShellFixture(t *testing.T, supervisor string) *initShellFixture {
+	t.Helper()
+	root := t.TempDir()
+	fx := &initShellFixture{
+		root:      root,
+		binDir:    filepath.Join(root, "bin"),
+		sysdDir:   filepath.Join(root, "sysd"),
+		rcDir:     filepath.Join(root, "rc"),
+		sysctlLog: filepath.Join(root, "sysctl.log"),
+		rcLog:     filepath.Join(root, "rc.log"),
+		rcuLog:    filepath.Join(root, "rcu.log"),
+	}
+	for _, d := range []string{
+		fx.binDir, fx.sysdDir, fx.rcDir, filepath.Join(root, "upload"),
+		filepath.Join(root, "usr/local/bin"), filepath.Join(root, "etc/rsmon-worker"),
+	} {
+		if err := os.MkdirAll(d, 0o755); err != nil {
+			t.Fatal(err)
+		}
+	}
+	writeExec(t, filepath.Join(fx.binDir, "systemctl"), systemctlStub)
+	writeExec(t, filepath.Join(fx.binDir, "rc-service"), rcServiceStub)
+	writeExec(t, filepath.Join(fx.binDir, "rc-update"), rcUpdateStub)
+
+	stage := filepath.Join(root, "usr/local/bin/rsmon-worker.stage")
+	writeExec(t, stage, workerStub)
+
+	envTmp := filepath.Join(root, "upload", "worker.env")
+	unitTmp := filepath.Join(root, "upload", "unit")
+	if err := os.WriteFile(envTmp, []byte(
+		"RSMON_URL=https://rsmon.ru\nRSMON_TOKEN=shell-test-token\nWORKER_HOST=127.0.0.1\nWORKER_PORT=27401\n",
+	), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	unitFile := filepath.Join(root, "etc/systemd/system/rsmon-worker.service")
+	unitMode := "0644"
+	if supervisor == "openrc" {
+		unitFile = filepath.Join(root, "etc/init.d/rsmon-worker")
+		unitMode = "0755"
+		if err := os.MkdirAll(filepath.Dir(unitFile), 0o755); err != nil {
+			t.Fatal(err)
+		}
+		if err := os.WriteFile(unitTmp, []byte("# fake openrc unit\n"), 0o755); err != nil {
+			t.Fatal(err)
+		}
+	} else {
+		if err := os.MkdirAll(filepath.Dir(unitFile), 0o755); err != nil {
+			t.Fatal(err)
+		}
+		if err := os.WriteFile(unitTmp, []byte("[Unit]\n# fake systemd unit\n"), 0o644); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	dataDir := filepath.Join(root, "var/lib/rsmon-worker")
+	fx.params = activateParams{
+		Stage:      stage,
+		Binary:     filepath.Join(root, "usr/local/bin/rsmon-worker"),
+		ConfigDir:  filepath.Join(root, "etc/rsmon-worker"),
+		EnvFile:    filepath.Join(root, "etc/rsmon-worker/worker.env"),
+		EnvTmp:     envTmp,
+		DataDir:    dataDir,
+		UnitTmp:    unitTmp,
+		UnitFile:   unitFile,
+		UnitMode:   unitMode,
+		UnitName:   "rsmon-worker.service",
+		RCName:     "rsmon-worker",
+		Supervisor: supervisor,
+		NoStart:    "0",
+	}
+	return fx
+}
+
+// run executes the rendered activation script with the stub init tools
+// first in PATH.
+func (fx *initShellFixture) run(t *testing.T) (string, error) {
+	t.Helper()
+	script := activateScript(fx.params)
+	cmd := exec.Command("sh", "-c", script)
+	cmd.Env = append(
+		os.Environ(),
+		"PATH="+fx.binDir+":"+os.Getenv("PATH"),
+		"FAKE_SYSTEMD_DIR="+fx.sysdDir,
+		"FAKE_RC_DIR="+fx.rcDir,
+		"SYSCTL_LOG="+fx.sysctlLog,
+		"RC_LOG="+fx.rcLog,
+		"RCU_LOG="+fx.rcuLog,
+	)
+	out, err := cmd.CombinedOutput()
+	return string(out), err
+}
+
+func (fx *initShellFixture) log(t *testing.T, path string) string {
+	t.Helper()
+	b, err := os.ReadFile(path)
+	if err != nil {
+		return ""
+	}
+	return string(b)
+}
+
+func (fx *initShellFixture) assertNoLeftovers(t *testing.T) {
+	t.Helper()
+	for _, name := range []string{".rsmon-backup", ".rsmon-activate.lock"} {
+		if _, err := os.Stat(filepath.Join(fx.params.DataDir, name)); !os.IsNotExist(err) {
+			t.Fatalf("%s left behind after activation", name)
+		}
+	}
+}
+
+// TestActivateShellSystemd runs the real activation script against a stub
+// systemd host and verifies the init-managed flow: the unit is installed
+// and enabled, restart goes through systemctl, and the health loop uses
+// `systemctl is-active` (no pid file is ever created).
+func TestActivateShellSystemd(t *testing.T) {
+	fx := newInitShellFixture(t, "systemd")
+	out, err := fx.run(t)
+	if err != nil {
+		t.Fatalf("systemd activation failed: %v\n%s", err, out)
+	}
+	if !strings.Contains(out, "rsmon-worker activated") {
+		t.Fatalf("no activation success line:\n%s", out)
+	}
+	unit, err := os.ReadFile(fx.params.UnitFile)
+	if err != nil {
+		t.Fatalf("unit not installed: %v", err)
+	}
+	if !strings.Contains(string(unit), "[Unit]") {
+		t.Fatalf("installed unit has wrong content: %q", unit)
+	}
+	st, err := os.Stat(fx.params.EnvFile)
+	if err != nil {
+		t.Fatalf("env not installed: %v", err)
+	}
+	if st.Mode().Perm() != 0o600 {
+		t.Fatalf("env mode = %v, want 0600", st.Mode().Perm())
+	}
+	log := fx.log(t, fx.sysctlLog)
+	for _, want := range []string{
+		"enable rsmon-worker.service",
+		"restart rsmon-worker.service",
+		"is-active --quiet rsmon-worker.service",
+	} {
+		if !strings.Contains(log, want) {
+			t.Fatalf("systemctl log missing %q:\n%s", want, log)
+		}
+	}
+	// The health loop used systemctl is-active, never the embedded
+	// supervisor's pid file.
+	if _, err := os.Stat(filepath.Join(fx.params.DataDir, "worker.pid")); !os.IsNotExist(err) {
+		t.Fatalf("systemd activation created a pid file; the health loop must use systemctl is-active")
+	}
+	if _, err := os.Stat(filepath.Join(fx.sysdDir, "rsmon-worker.service.enabled")); err != nil {
+		t.Fatalf("unit not enabled: %v", err)
+	}
+	fx.assertNoLeftovers(t)
+}
+
+// TestActivateShellOpenRC runs the real activation script against a stub
+// OpenRC host: the init script is installed executable, the runlevel
+// enablement and rc-service restart/status are used, and the health loop
+// goes through rc-service (no pid file).
+func TestActivateShellOpenRC(t *testing.T) {
+	fx := newInitShellFixture(t, "openrc")
+	out, err := fx.run(t)
+	if err != nil {
+		t.Fatalf("openrc activation failed: %v\n%s", err, out)
+	}
+	if !strings.Contains(out, "rsmon-worker activated") {
+		t.Fatalf("no activation success line:\n%s", out)
+	}
+	st, err := os.Stat(fx.params.UnitFile)
+	if err != nil {
+		t.Fatalf("openrc init not installed: %v", err)
+	}
+	if st.Mode().Perm() != 0o755 {
+		t.Fatalf("openrc init mode = %v, want 0755", st.Mode().Perm())
+	}
+	rcLog := fx.log(t, fx.rcLog)
+	for _, want := range []string{"rsmon-worker restart", "rsmon-worker status"} {
+		if !strings.Contains(rcLog, want) {
+			t.Fatalf("rc-service log missing %q:\n%s", want, rcLog)
+		}
+	}
+	rcuLog := fx.log(t, fx.rcuLog)
+	if !strings.Contains(rcuLog, "add rsmon-worker default") {
+		t.Fatalf("rc-update add not issued:\n%s", rcuLog)
+	}
+	if _, err := os.Stat(filepath.Join(fx.rcDir, "rsmon-worker.enabled")); err != nil {
+		t.Fatalf("service not enabled in the runlevel: %v", err)
+	}
+	if _, err := os.Stat(filepath.Join(fx.params.DataDir, "worker.pid")); !os.IsNotExist(err) {
+		t.Fatalf("openrc activation created a pid file; the health loop must use rc-service status")
+	}
+	fx.assertNoLeftovers(t)
+}
+
+// TestActivateShellSystemdRollbackFreshDisablesUnit proves a failed
+// activation on a fresh host disables the newly-enabled unit and removes
+// the unit file (no dangling enablement).
+func TestActivateShellSystemdRollbackFreshDisablesUnit(t *testing.T) {
+	fx := newInitShellFixture(t, "systemd")
+	if err := os.WriteFile(filepath.Join(fx.sysdDir, "fail-restart"), []byte(""), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	out, err := fx.run(t)
+	if err == nil {
+		t.Fatal("activation succeeded despite restart failure")
+	}
+	if !strings.Contains(out, "start failure") {
+		t.Fatalf("no start-failure classification:\n%s", out)
+	}
+	log := fx.log(t, fx.sysctlLog)
+	if !strings.Contains(log, "disable rsmon-worker.service") {
+		t.Fatalf("fresh rollback did not disable the unit:\n%s", log)
+	}
+	if _, err := os.Stat(fx.params.UnitFile); !os.IsNotExist(err) {
+		t.Fatal("unit file left behind after fresh rollback")
+	}
+	if _, err := os.Stat(filepath.Join(fx.sysdDir, "rsmon-worker.service.enabled")); !os.IsNotExist(err) {
+		t.Fatal("unit still enabled after fresh rollback")
+	}
+	fx.assertNoLeftovers(t)
+}
+
+// TestActivateShellSystemdRollbackRestoresEnableState proves a failed
+// activation on a host with a prior enabled unit restores the unit
+// byte-for-byte and re-applies the prior enabled state.
+func TestActivateShellSystemdRollbackRestoresEnableState(t *testing.T) {
+	fx := newInitShellFixture(t, "systemd")
+	if err := os.WriteFile(fx.params.UnitFile, []byte("PRIOR-UNIT-CONTENT"), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(filepath.Join(fx.sysdDir, "rsmon-worker.service.enabled"), []byte(""), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(filepath.Join(fx.sysdDir, "fail-restart"), []byte(""), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	out, err := fx.run(t)
+	if err == nil {
+		t.Fatal("activation succeeded despite restart failure")
+	}
+	if !strings.Contains(out, "start failure") {
+		t.Fatalf("no start-failure classification:\n%s", out)
+	}
+	unit, err := os.ReadFile(fx.params.UnitFile)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if string(unit) != "PRIOR-UNIT-CONTENT" {
+		t.Fatalf("prior unit not restored: %q", unit)
+	}
+	log := fx.log(t, fx.sysctlLog)
+	if !strings.Contains(log, "enable rsmon-worker.service") {
+		t.Fatalf("prior enabled state not re-applied:\n%s", log)
+	}
+	if _, err := os.Stat(filepath.Join(fx.sysdDir, "rsmon-worker.service.enabled")); err != nil {
+		t.Fatalf("unit not re-enabled after rollback: %v", err)
+	}
+	fx.assertNoLeftovers(t)
+}
+
+// TestActivateShellStaleLockBroken proves a leftover lock from a killed
+// run (dead pid) is broken automatically and the activation proceeds.
+func TestActivateShellStaleLockBroken(t *testing.T) {
+	fx := newInitShellFixture(t, "systemd")
+	lockDir := filepath.Join(fx.params.DataDir, ".rsmon-activate.lock")
+	if err := os.MkdirAll(lockDir, 0o700); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(filepath.Join(lockDir, "pid"), []byte("999999\n"), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	out, err := fx.run(t)
+	if err != nil {
+		t.Fatalf("activation failed with a stale lock: %v\n%s", err, out)
+	}
+	if !strings.Contains(out, "rsmon-worker activated") {
+		t.Fatalf("no activation success line:\n%s", out)
+	}
+	if _, err := os.Stat(lockDir); !os.IsNotExist(err) {
+		t.Fatal("lock not released after activation")
+	}
+}
+
+// TestActivateShellRecoveryFromInterruptedBackup proves the interrupted-run
+// recovery: a leftover backup marker from a killed activation is restored
+// before the fresh run, so a later validation failure still leaves the
+// recovered prior install in place.
+func TestActivateShellRecoveryFromInterruptedBackup(t *testing.T) {
+	fx := newInitShellFixture(t, "none")
+	backupDir := filepath.Join(fx.params.DataDir, ".rsmon-backup")
+	if err := os.MkdirAll(backupDir, 0o700); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(filepath.Join(backupDir, "binary"), []byte("RECOVERED-BINARY"), 0o755); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(filepath.Join(backupDir, "worker.env"), []byte("RSMON_URL=https://rsmon.ru\nRSMON_TOKEN=test\n"), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(filepath.Join(backupDir, ".marker"), []byte("0"), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	// Corrupt the stage so the fresh activation fails at validation (before
+	// the fresh snapshot), proving the recovered files were already in place.
+	if err := os.WriteFile(fx.params.Stage, []byte("not a script"), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	out, err := fx.run(t)
+	if err == nil {
+		t.Fatal("activation succeeded with a corrupt stage")
+	}
+	if !strings.Contains(out, "recovering interrupted activation") {
+		t.Fatalf("recovery did not run:\n%s", out)
+	}
+	b, err := os.ReadFile(fx.params.Binary)
+	if err != nil {
+		t.Fatalf("recovered binary not restored: %v", err)
+	}
+	if string(b) != "RECOVERED-BINARY" {
+		t.Fatalf("recovered binary content = %q", b)
+	}
+	if _, err := os.Stat(backupDir); !os.IsNotExist(err) {
+		t.Fatal("backup not consumed by recovery")
+	}
+}