worker/internal/installer/sourceactivate_shell_test.go

package installer

import (
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"testing"
)

// This file exercises the rendered activation script against stub
// systemd/OpenRC implementations ("targeted shell fixture tests"). Real
// systemd and OpenRC cannot run inside the Docker/OpenSSH harness (the
// container PID 1 is sshd), so these tests execute the actual remote
// script on the host with fake `systemctl`, `rc-service`, and `rc-update`
// binaries that simulate an init-managed host. This covers the unit
// install/enable, the supervisor-driven restart, the `is-active`-based
// health loop, the enable-state rollback, the activation lock, and the
// interrupted-run recovery paths that the E2E fixtures cannot reach.
//
// Residual limitation: the stub init tools verify command flow and state
// transitions, not the real systemd/OpenRC unit semantics; a real
// init-system smoke test remains out of scope for the container harness.

// systemctlStub records every invocation and simulates unit state in a
// fake state directory: `is-active`/`is-enabled` consult marker files,
// `restart` marks the unit active (or fails when the fail-restart marker
// is present), and `enable`/`disable` toggle the enabled marker.
const systemctlStub = `#!/bin/sh
echo "$*" >> "$SYSCTL_LOG"
action="$1"
shift
unit=""
for a in "$@"; do
    case "$a" in --*) continue ;; esac
    unit="$a"
    break
done
case "$action" in
    is-active)  [ -f "$FAKE_SYSTEMD_DIR/$unit.active" ] || exit 3 ;;
    is-enabled) [ -f "$FAKE_SYSTEMD_DIR/$unit.enabled" ] || exit 1 ;;
    enable)     touch "$FAKE_SYSTEMD_DIR/$unit.enabled"; exit 0 ;;
    disable)    rm -f "$FAKE_SYSTEMD_DIR/$unit.enabled"; exit 0 ;;
    daemon-reload) exit 0 ;;
    restart)    [ -f "$FAKE_SYSTEMD_DIR/fail-restart" ] && exit 1
                touch "$FAKE_SYSTEMD_DIR/$unit.active"; exit 0 ;;
    stop)       rm -f "$FAKE_SYSTEMD_DIR/$unit.active"; exit 0 ;;
    *) exit 0 ;;
esac
`

// rcServiceStub simulates OpenRC service state; rcUpdateStub simulates
// runlevel enablement.
const rcServiceStub = `#!/bin/sh
echo "$*" >> "$RC_LOG"
svc="$1"
action="$2"
case "$action" in
    restart) [ -f "$FAKE_RC_DIR/fail-restart" ] && exit 1
             touch "$FAKE_RC_DIR/$svc.active"; exit 0 ;;
    status)  [ -f "$FAKE_RC_DIR/$svc.active" ] || exit 1 ;;
    stop)    rm -f "$FAKE_RC_DIR/$svc.active"; exit 0 ;;
    *) exit 0 ;;
esac
`

const rcUpdateStub = `#!/bin/sh
echo "$*" >> "$RCU_LOG"
case "$1" in
    add) touch "$FAKE_RC_DIR/$2.enabled"; exit 0 ;;
    del) rm -f "$FAKE_RC_DIR/$2.enabled"; exit 0 ;;
    *) exit 0 ;;
esac
`

// workerStub stands in for the built worker binary: it satisfies the
// `--version` validation and the `/healthz` liveness probe.
const workerStub = `#!/bin/sh
if [ "$1" = "--version" ]; then
    echo "rsmon-worker version=dev commit=deadbeefdead buildDate=2026-01-01T00:00:00Z"
    exit 0
fi
if [ "$1" = "liveness" ]; then
    exit 0
fi
exit 1
`

// initShellFixture builds a temp "host" with stub init tools and a stub
// worker, plus the uploaded env/unit the activation script consumes.
type initShellFixture struct {
	root    string
	binDir  string
	sysdDir string
	rcDir   string

	sysctlLog string
	rcLog     string
	rcuLog    string

	params activateParams
}

func writeExec(t *testing.T, path, content string) {
	t.Helper()
	if err := os.WriteFile(path, []byte(content), 0o700); err != nil {
		t.Fatal(err)
	}
}

func newInitShellFixture(t *testing.T, supervisor string) *initShellFixture {
	t.Helper()
	root := t.TempDir()
	fx := &initShellFixture{
		root:      root,
		binDir:    filepath.Join(root, "bin"),
		sysdDir:   filepath.Join(root, "sysd"),
		rcDir:     filepath.Join(root, "rc"),
		sysctlLog: filepath.Join(root, "sysctl.log"),
		rcLog:     filepath.Join(root, "rc.log"),
		rcuLog:    filepath.Join(root, "rcu.log"),
	}
	for _, d := range []string{
		fx.binDir, fx.sysdDir, fx.rcDir, filepath.Join(root, "upload"),
		filepath.Join(root, "usr/local/bin"), filepath.Join(root, "etc/rsmon-worker"),
	} {
		if err := os.MkdirAll(d, 0o755); err != nil {
			t.Fatal(err)
		}
	}
	writeExec(t, filepath.Join(fx.binDir, "systemctl"), systemctlStub)
	writeExec(t, filepath.Join(fx.binDir, "rc-service"), rcServiceStub)
	writeExec(t, filepath.Join(fx.binDir, "rc-update"), rcUpdateStub)

	stage := filepath.Join(root, "usr/local/bin/rsmon-worker.stage")
	writeExec(t, stage, workerStub)

	envTmp := filepath.Join(root, "upload", "worker.env")
	unitTmp := filepath.Join(root, "upload", "unit")
	if err := os.WriteFile(envTmp, []byte(
		"RSMON_URL=https://rsmon.ru\nRSMON_TOKEN=shell-test-token\nWORKER_HOST=127.0.0.1\nWORKER_PORT=27401\n",
	), 0o600); err != nil {
		t.Fatal(err)
	}
	unitFile := filepath.Join(root, "etc/systemd/system/rsmon-worker.service")
	unitMode := "0644"
	if supervisor == "openrc" {
		unitFile = filepath.Join(root, "etc/init.d/rsmon-worker")
		unitMode = "0755"
		if err := os.MkdirAll(filepath.Dir(unitFile), 0o755); err != nil {
			t.Fatal(err)
		}
		if err := os.WriteFile(unitTmp, []byte("# fake openrc unit\n"), 0o755); err != nil {
			t.Fatal(err)
		}
	} else {
		if err := os.MkdirAll(filepath.Dir(unitFile), 0o755); err != nil {
			t.Fatal(err)
		}
		if err := os.WriteFile(unitTmp, []byte("[Unit]\n# fake systemd unit\n"), 0o644); err != nil {
			t.Fatal(err)
		}
	}

	dataDir := filepath.Join(root, "var/lib/rsmon-worker")
	fx.params = activateParams{
		Stage:      stage,
		Binary:     filepath.Join(root, "usr/local/bin/rsmon-worker"),
		ConfigDir:  filepath.Join(root, "etc/rsmon-worker"),
		EnvFile:    filepath.Join(root, "etc/rsmon-worker/worker.env"),
		EnvTmp:     envTmp,
		DataDir:    dataDir,
		UnitTmp:    unitTmp,
		UnitFile:   unitFile,
		UnitMode:   unitMode,
		UnitName:   "rsmon-worker.service",
		RCName:     "rsmon-worker",
		Supervisor: supervisor,
		NoStart:    "0",
	}
	return fx
}

// run executes the rendered activation script with the stub init tools
// first in PATH.
func (fx *initShellFixture) run(t *testing.T) (string, error) {
	t.Helper()
	script := activateScript(fx.params)
	cmd := exec.Command("sh", "-c", script)
	cmd.Env = append(
		os.Environ(),
		"PATH="+fx.binDir+":"+os.Getenv("PATH"),
		"FAKE_SYSTEMD_DIR="+fx.sysdDir,
		"FAKE_RC_DIR="+fx.rcDir,
		"SYSCTL_LOG="+fx.sysctlLog,
		"RC_LOG="+fx.rcLog,
		"RCU_LOG="+fx.rcuLog,
	)
	out, err := cmd.CombinedOutput()
	return string(out), err
}

func (fx *initShellFixture) log(t *testing.T, path string) string {
	t.Helper()
	b, err := os.ReadFile(path)
	if err != nil {
		return ""
	}
	return string(b)
}

func (fx *initShellFixture) assertNoLeftovers(t *testing.T) {
	t.Helper()
	for _, name := range []string{".rsmon-backup", ".rsmon-activate.lock"} {
		if _, err := os.Stat(filepath.Join(fx.params.DataDir, name)); !os.IsNotExist(err) {
			t.Fatalf("%s left behind after activation", name)
		}
	}
}

// TestActivateShellSystemd runs the real activation script against a stub
// systemd host and verifies the init-managed flow: the unit is installed
// and enabled, restart goes through systemctl, and the health loop uses
// `systemctl is-active` (no pid file is ever created).
func TestActivateShellSystemd(t *testing.T) {
	fx := newInitShellFixture(t, "systemd")
	out, err := fx.run(t)
	if err != nil {
		t.Fatalf("systemd activation failed: %v\n%s", err, out)
	}
	if !strings.Contains(out, "rsmon-worker activated") {
		t.Fatalf("no activation success line:\n%s", out)
	}
	unit, err := os.ReadFile(fx.params.UnitFile)
	if err != nil {
		t.Fatalf("unit not installed: %v", err)
	}
	if !strings.Contains(string(unit), "[Unit]") {
		t.Fatalf("installed unit has wrong content: %q", unit)
	}
	st, err := os.Stat(fx.params.EnvFile)
	if err != nil {
		t.Fatalf("env not installed: %v", err)
	}
	if st.Mode().Perm() != 0o600 {
		t.Fatalf("env mode = %v, want 0600", st.Mode().Perm())
	}
	log := fx.log(t, fx.sysctlLog)
	for _, want := range []string{
		"enable rsmon-worker.service",
		"restart rsmon-worker.service",
		"is-active --quiet rsmon-worker.service",
	} {
		if !strings.Contains(log, want) {
			t.Fatalf("systemctl log missing %q:\n%s", want, log)
		}
	}
	// The health loop used systemctl is-active, never the embedded
	// supervisor's pid file.
	if _, err := os.Stat(filepath.Join(fx.params.DataDir, "worker.pid")); !os.IsNotExist(err) {
		t.Fatalf("systemd activation created a pid file; the health loop must use systemctl is-active")
	}
	if _, err := os.Stat(filepath.Join(fx.sysdDir, "rsmon-worker.service.enabled")); err != nil {
		t.Fatalf("unit not enabled: %v", err)
	}
	fx.assertNoLeftovers(t)
}

// TestActivateShellOpenRC runs the real activation script against a stub
// OpenRC host: the init script is installed executable, the runlevel
// enablement and rc-service restart/status are used, and the health loop
// goes through rc-service (no pid file).
func TestActivateShellOpenRC(t *testing.T) {
	fx := newInitShellFixture(t, "openrc")
	out, err := fx.run(t)
	if err != nil {
		t.Fatalf("openrc activation failed: %v\n%s", err, out)
	}
	if !strings.Contains(out, "rsmon-worker activated") {
		t.Fatalf("no activation success line:\n%s", out)
	}
	st, err := os.Stat(fx.params.UnitFile)
	if err != nil {
		t.Fatalf("openrc init not installed: %v", err)
	}
	if st.Mode().Perm() != 0o755 {
		t.Fatalf("openrc init mode = %v, want 0755", st.Mode().Perm())
	}
	rcLog := fx.log(t, fx.rcLog)
	for _, want := range []string{"rsmon-worker restart", "rsmon-worker status"} {
		if !strings.Contains(rcLog, want) {
			t.Fatalf("rc-service log missing %q:\n%s", want, rcLog)
		}
	}
	rcuLog := fx.log(t, fx.rcuLog)
	if !strings.Contains(rcuLog, "add rsmon-worker default") {
		t.Fatalf("rc-update add not issued:\n%s", rcuLog)
	}
	if _, err := os.Stat(filepath.Join(fx.rcDir, "rsmon-worker.enabled")); err != nil {
		t.Fatalf("service not enabled in the runlevel: %v", err)
	}
	if _, err := os.Stat(filepath.Join(fx.params.DataDir, "worker.pid")); !os.IsNotExist(err) {
		t.Fatalf("openrc activation created a pid file; the health loop must use rc-service status")
	}
	fx.assertNoLeftovers(t)
}

// TestActivateShellSystemdRollbackFreshDisablesUnit proves a failed
// activation on a fresh host disables the newly-enabled unit and removes
// the unit file (no dangling enablement).
func TestActivateShellSystemdRollbackFreshDisablesUnit(t *testing.T) {
	fx := newInitShellFixture(t, "systemd")
	if err := os.WriteFile(filepath.Join(fx.sysdDir, "fail-restart"), []byte(""), 0o644); err != nil {
		t.Fatal(err)
	}
	out, err := fx.run(t)
	if err == nil {
		t.Fatal("activation succeeded despite restart failure")
	}
	if !strings.Contains(out, "start failure") {
		t.Fatalf("no start-failure classification:\n%s", out)
	}
	log := fx.log(t, fx.sysctlLog)
	if !strings.Contains(log, "disable rsmon-worker.service") {
		t.Fatalf("fresh rollback did not disable the unit:\n%s", log)
	}
	if _, err := os.Stat(fx.params.UnitFile); !os.IsNotExist(err) {
		t.Fatal("unit file left behind after fresh rollback")
	}
	if _, err := os.Stat(filepath.Join(fx.sysdDir, "rsmon-worker.service.enabled")); !os.IsNotExist(err) {
		t.Fatal("unit still enabled after fresh rollback")
	}
	fx.assertNoLeftovers(t)
}

// TestActivateShellSystemdRollbackRestoresEnableState proves a failed
// activation on a host with a prior enabled unit restores the unit
// byte-for-byte and re-applies the prior enabled state.
func TestActivateShellSystemdRollbackRestoresEnableState(t *testing.T) {
	fx := newInitShellFixture(t, "systemd")
	if err := os.WriteFile(fx.params.UnitFile, []byte("PRIOR-UNIT-CONTENT"), 0o644); err != nil {
		t.Fatal(err)
	}
	if err := os.WriteFile(filepath.Join(fx.sysdDir, "rsmon-worker.service.enabled"), []byte(""), 0o644); err != nil {
		t.Fatal(err)
	}
	if err := os.WriteFile(filepath.Join(fx.sysdDir, "fail-restart"), []byte(""), 0o644); err != nil {
		t.Fatal(err)
	}
	out, err := fx.run(t)
	if err == nil {
		t.Fatal("activation succeeded despite restart failure")
	}
	if !strings.Contains(out, "start failure") {
		t.Fatalf("no start-failure classification:\n%s", out)
	}
	unit, err := os.ReadFile(fx.params.UnitFile)
	if err != nil {
		t.Fatal(err)
	}
	if string(unit) != "PRIOR-UNIT-CONTENT" {
		t.Fatalf("prior unit not restored: %q", unit)
	}
	log := fx.log(t, fx.sysctlLog)
	if !strings.Contains(log, "enable rsmon-worker.service") {
		t.Fatalf("prior enabled state not re-applied:\n%s", log)
	}
	if _, err := os.Stat(filepath.Join(fx.sysdDir, "rsmon-worker.service.enabled")); err != nil {
		t.Fatalf("unit not re-enabled after rollback: %v", err)
	}
	fx.assertNoLeftovers(t)
}

// TestActivateShellStaleLockBroken proves a leftover lock from a killed
// run (dead pid) is broken automatically and the activation proceeds.
func TestActivateShellStaleLockBroken(t *testing.T) {
	fx := newInitShellFixture(t, "systemd")
	lockDir := filepath.Join(fx.params.DataDir, ".rsmon-activate.lock")
	if err := os.MkdirAll(lockDir, 0o700); err != nil {
		t.Fatal(err)
	}
	if err := os.WriteFile(filepath.Join(lockDir, "pid"), []byte("999999\n"), 0o644); err != nil {
		t.Fatal(err)
	}
	out, err := fx.run(t)
	if err != nil {
		t.Fatalf("activation failed with a stale lock: %v\n%s", err, out)
	}
	if !strings.Contains(out, "rsmon-worker activated") {
		t.Fatalf("no activation success line:\n%s", out)
	}
	if _, err := os.Stat(lockDir); !os.IsNotExist(err) {
		t.Fatal("lock not released after activation")
	}
}

// TestActivateShellRecoveryFromInterruptedBackup proves the interrupted-run
// recovery: a leftover backup marker from a killed activation is restored
// before the fresh run, so a later validation failure still leaves the
// recovered prior install in place.
func TestActivateShellRecoveryFromInterruptedBackup(t *testing.T) {
	fx := newInitShellFixture(t, "none")
	backupDir := filepath.Join(fx.params.DataDir, ".rsmon-backup")
	if err := os.MkdirAll(backupDir, 0o700); err != nil {
		t.Fatal(err)
	}
	if err := os.WriteFile(filepath.Join(backupDir, "binary"), []byte("RECOVERED-BINARY"), 0o755); err != nil {
		t.Fatal(err)
	}
	if err := os.WriteFile(filepath.Join(backupDir, "worker.env"), []byte("RSMON_URL=https://rsmon.ru\nRSMON_TOKEN=test\n"), 0o600); err != nil {
		t.Fatal(err)
	}
	if err := os.WriteFile(filepath.Join(backupDir, ".marker"), []byte("0"), 0o600); err != nil {
		t.Fatal(err)
	}
	// Corrupt the stage so the fresh activation fails at validation (before
	// the fresh snapshot), proving the recovered files were already in place.
	if err := os.WriteFile(fx.params.Stage, []byte("not a script"), 0o644); err != nil {
		t.Fatal(err)
	}
	out, err := fx.run(t)
	if err == nil {
		t.Fatal("activation succeeded with a corrupt stage")
	}
	if !strings.Contains(out, "recovering interrupted activation") {
		t.Fatalf("recovery did not run:\n%s", out)
	}
	b, err := os.ReadFile(fx.params.Binary)
	if err != nil {
		t.Fatalf("recovered binary not restored: %v", err)
	}
	if string(b) != "RECOVERED-BINARY" {
		t.Fatalf("recovered binary content = %q", b)
	}
	if _, err := os.Stat(backupDir); !os.IsNotExist(err) {
		t.Fatal("backup not consumed by recovery")
	}
}