package installer import ( "os" "os/exec" "path/filepath" "strings" "testing" ) // This file exercises the rendered activation script against stub // systemd/OpenRC implementations ("targeted shell fixture tests"). Real // systemd and OpenRC cannot run inside the Docker/OpenSSH harness (the // container PID 1 is sshd), so these tests execute the actual remote // script on the host with fake `systemctl`, `rc-service`, and `rc-update` // binaries that simulate an init-managed host. This covers the unit // install/enable, the supervisor-driven restart, the `is-active`-based // health loop, the enable-state rollback, the activation lock, and the // interrupted-run recovery paths that the E2E fixtures cannot reach. // // Residual limitation: the stub init tools verify command flow and state // transitions, not the real systemd/OpenRC unit semantics; a real // init-system smoke test remains out of scope for the container harness. // systemctlStub records every invocation and simulates unit state in a // fake state directory: `is-active`/`is-enabled` consult marker files, // `restart` marks the unit active (or fails when the fail-restart marker // is present), and `enable`/`disable` toggle the enabled marker. const systemctlStub = `#!/bin/sh echo "$*" >> "$SYSCTL_LOG" action="$1" shift unit="" for a in "$@"; do case "$a" in --*) continue ;; esac unit="$a" break done case "$action" in is-active) [ -f "$FAKE_SYSTEMD_DIR/$unit.active" ] || exit 3 ;; is-enabled) [ -f "$FAKE_SYSTEMD_DIR/$unit.enabled" ] || exit 1 ;; enable) touch "$FAKE_SYSTEMD_DIR/$unit.enabled"; exit 0 ;; disable) rm -f "$FAKE_SYSTEMD_DIR/$unit.enabled"; exit 0 ;; daemon-reload) exit 0 ;; restart) [ -f "$FAKE_SYSTEMD_DIR/fail-restart" ] && exit 1 touch "$FAKE_SYSTEMD_DIR/$unit.active"; exit 0 ;; stop) rm -f "$FAKE_SYSTEMD_DIR/$unit.active"; exit 0 ;; *) exit 0 ;; esac ` // rcServiceStub simulates OpenRC service state; rcUpdateStub simulates // runlevel enablement. const rcServiceStub = `#!/bin/sh echo "$*" >> "$RC_LOG" svc="$1" action="$2" case "$action" in restart) [ -f "$FAKE_RC_DIR/fail-restart" ] && exit 1 touch "$FAKE_RC_DIR/$svc.active"; exit 0 ;; status) [ -f "$FAKE_RC_DIR/$svc.active" ] || exit 1 ;; stop) rm -f "$FAKE_RC_DIR/$svc.active"; exit 0 ;; *) exit 0 ;; esac ` const rcUpdateStub = `#!/bin/sh echo "$*" >> "$RCU_LOG" case "$1" in add) touch "$FAKE_RC_DIR/$2.enabled"; exit 0 ;; del) rm -f "$FAKE_RC_DIR/$2.enabled"; exit 0 ;; *) exit 0 ;; esac ` // workerStub stands in for the built worker binary: it satisfies the // `--version` validation and the `/healthz` liveness probe. const workerStub = `#!/bin/sh if [ "$1" = "--version" ]; then echo "rsmon-worker version=dev commit=deadbeefdead buildDate=2026-01-01T00:00:00Z" exit 0 fi if [ "$1" = "liveness" ]; then exit 0 fi exit 1 ` // initShellFixture builds a temp "host" with stub init tools and a stub // worker, plus the uploaded env/unit the activation script consumes. type initShellFixture struct { root string binDir string sysdDir string rcDir string sysctlLog string rcLog string rcuLog string params activateParams } func writeExec(t *testing.T, path, content string) { t.Helper() if err := os.WriteFile(path, []byte(content), 0o700); err != nil { t.Fatal(err) } } func newInitShellFixture(t *testing.T, supervisor string) *initShellFixture { t.Helper() root := t.TempDir() fx := &initShellFixture{ root: root, binDir: filepath.Join(root, "bin"), sysdDir: filepath.Join(root, "sysd"), rcDir: filepath.Join(root, "rc"), sysctlLog: filepath.Join(root, "sysctl.log"), rcLog: filepath.Join(root, "rc.log"), rcuLog: filepath.Join(root, "rcu.log"), } for _, d := range []string{ fx.binDir, fx.sysdDir, fx.rcDir, filepath.Join(root, "upload"), filepath.Join(root, "usr/local/bin"), filepath.Join(root, "etc/rsmon-worker"), } { if err := os.MkdirAll(d, 0o755); err != nil { t.Fatal(err) } } writeExec(t, filepath.Join(fx.binDir, "systemctl"), systemctlStub) writeExec(t, filepath.Join(fx.binDir, "rc-service"), rcServiceStub) writeExec(t, filepath.Join(fx.binDir, "rc-update"), rcUpdateStub) stage := filepath.Join(root, "usr/local/bin/rsmon-worker.stage") writeExec(t, stage, workerStub) envTmp := filepath.Join(root, "upload", "worker.env") unitTmp := filepath.Join(root, "upload", "unit") if err := os.WriteFile(envTmp, []byte( "RSMON_URL=https://rsmon.ru\nRSMON_TOKEN=shell-test-token\nWORKER_HOST=127.0.0.1\nWORKER_PORT=27401\n", ), 0o600); err != nil { t.Fatal(err) } unitFile := filepath.Join(root, "etc/systemd/system/rsmon-worker.service") unitMode := "0644" if supervisor == "openrc" { unitFile = filepath.Join(root, "etc/init.d/rsmon-worker") unitMode = "0755" if err := os.MkdirAll(filepath.Dir(unitFile), 0o755); err != nil { t.Fatal(err) } if err := os.WriteFile(unitTmp, []byte("# fake openrc unit\n"), 0o755); err != nil { t.Fatal(err) } } else { if err := os.MkdirAll(filepath.Dir(unitFile), 0o755); err != nil { t.Fatal(err) } if err := os.WriteFile(unitTmp, []byte("[Unit]\n# fake systemd unit\n"), 0o644); err != nil { t.Fatal(err) } } dataDir := filepath.Join(root, "var/lib/rsmon-worker") fx.params = activateParams{ Stage: stage, Binary: filepath.Join(root, "usr/local/bin/rsmon-worker"), ConfigDir: filepath.Join(root, "etc/rsmon-worker"), EnvFile: filepath.Join(root, "etc/rsmon-worker/worker.env"), EnvTmp: envTmp, DataDir: dataDir, UnitTmp: unitTmp, UnitFile: unitFile, UnitMode: unitMode, UnitName: "rsmon-worker.service", RCName: "rsmon-worker", Supervisor: supervisor, NoStart: "0", } return fx } // run executes the rendered activation script with the stub init tools // first in PATH. func (fx *initShellFixture) run(t *testing.T) (string, error) { t.Helper() script := activateScript(fx.params) cmd := exec.Command("sh", "-c", script) cmd.Env = append( os.Environ(), "PATH="+fx.binDir+":"+os.Getenv("PATH"), "FAKE_SYSTEMD_DIR="+fx.sysdDir, "FAKE_RC_DIR="+fx.rcDir, "SYSCTL_LOG="+fx.sysctlLog, "RC_LOG="+fx.rcLog, "RCU_LOG="+fx.rcuLog, ) out, err := cmd.CombinedOutput() return string(out), err } func (fx *initShellFixture) log(t *testing.T, path string) string { t.Helper() b, err := os.ReadFile(path) if err != nil { return "" } return string(b) } func (fx *initShellFixture) assertNoLeftovers(t *testing.T) { t.Helper() for _, name := range []string{".rsmon-backup", ".rsmon-activate.lock"} { if _, err := os.Stat(filepath.Join(fx.params.DataDir, name)); !os.IsNotExist(err) { t.Fatalf("%s left behind after activation", name) } } } // TestActivateShellSystemd runs the real activation script against a stub // systemd host and verifies the init-managed flow: the unit is installed // and enabled, restart goes through systemctl, and the health loop uses // `systemctl is-active` (no pid file is ever created). func TestActivateShellSystemd(t *testing.T) { fx := newInitShellFixture(t, "systemd") out, err := fx.run(t) if err != nil { t.Fatalf("systemd activation failed: %v\n%s", err, out) } if !strings.Contains(out, "rsmon-worker activated") { t.Fatalf("no activation success line:\n%s", out) } unit, err := os.ReadFile(fx.params.UnitFile) if err != nil { t.Fatalf("unit not installed: %v", err) } if !strings.Contains(string(unit), "[Unit]") { t.Fatalf("installed unit has wrong content: %q", unit) } st, err := os.Stat(fx.params.EnvFile) if err != nil { t.Fatalf("env not installed: %v", err) } if st.Mode().Perm() != 0o600 { t.Fatalf("env mode = %v, want 0600", st.Mode().Perm()) } log := fx.log(t, fx.sysctlLog) for _, want := range []string{ "enable rsmon-worker.service", "restart rsmon-worker.service", "is-active --quiet rsmon-worker.service", } { if !strings.Contains(log, want) { t.Fatalf("systemctl log missing %q:\n%s", want, log) } } // The health loop used systemctl is-active, never the embedded // supervisor's pid file. if _, err := os.Stat(filepath.Join(fx.params.DataDir, "worker.pid")); !os.IsNotExist(err) { t.Fatalf("systemd activation created a pid file; the health loop must use systemctl is-active") } if _, err := os.Stat(filepath.Join(fx.sysdDir, "rsmon-worker.service.enabled")); err != nil { t.Fatalf("unit not enabled: %v", err) } fx.assertNoLeftovers(t) } // TestActivateShellOpenRC runs the real activation script against a stub // OpenRC host: the init script is installed executable, the runlevel // enablement and rc-service restart/status are used, and the health loop // goes through rc-service (no pid file). func TestActivateShellOpenRC(t *testing.T) { fx := newInitShellFixture(t, "openrc") out, err := fx.run(t) if err != nil { t.Fatalf("openrc activation failed: %v\n%s", err, out) } if !strings.Contains(out, "rsmon-worker activated") { t.Fatalf("no activation success line:\n%s", out) } st, err := os.Stat(fx.params.UnitFile) if err != nil { t.Fatalf("openrc init not installed: %v", err) } if st.Mode().Perm() != 0o755 { t.Fatalf("openrc init mode = %v, want 0755", st.Mode().Perm()) } rcLog := fx.log(t, fx.rcLog) for _, want := range []string{"rsmon-worker restart", "rsmon-worker status"} { if !strings.Contains(rcLog, want) { t.Fatalf("rc-service log missing %q:\n%s", want, rcLog) } } rcuLog := fx.log(t, fx.rcuLog) if !strings.Contains(rcuLog, "add rsmon-worker default") { t.Fatalf("rc-update add not issued:\n%s", rcuLog) } if _, err := os.Stat(filepath.Join(fx.rcDir, "rsmon-worker.enabled")); err != nil { t.Fatalf("service not enabled in the runlevel: %v", err) } if _, err := os.Stat(filepath.Join(fx.params.DataDir, "worker.pid")); !os.IsNotExist(err) { t.Fatalf("openrc activation created a pid file; the health loop must use rc-service status") } fx.assertNoLeftovers(t) } // TestActivateShellSystemdRollbackFreshDisablesUnit proves a failed // activation on a fresh host disables the newly-enabled unit and removes // the unit file (no dangling enablement). func TestActivateShellSystemdRollbackFreshDisablesUnit(t *testing.T) { fx := newInitShellFixture(t, "systemd") if err := os.WriteFile(filepath.Join(fx.sysdDir, "fail-restart"), []byte(""), 0o644); err != nil { t.Fatal(err) } out, err := fx.run(t) if err == nil { t.Fatal("activation succeeded despite restart failure") } if !strings.Contains(out, "start failure") { t.Fatalf("no start-failure classification:\n%s", out) } log := fx.log(t, fx.sysctlLog) if !strings.Contains(log, "disable rsmon-worker.service") { t.Fatalf("fresh rollback did not disable the unit:\n%s", log) } if _, err := os.Stat(fx.params.UnitFile); !os.IsNotExist(err) { t.Fatal("unit file left behind after fresh rollback") } if _, err := os.Stat(filepath.Join(fx.sysdDir, "rsmon-worker.service.enabled")); !os.IsNotExist(err) { t.Fatal("unit still enabled after fresh rollback") } fx.assertNoLeftovers(t) } // TestActivateShellSystemdRollbackRestoresEnableState proves a failed // activation on a host with a prior enabled unit restores the unit // byte-for-byte and re-applies the prior enabled state. func TestActivateShellSystemdRollbackRestoresEnableState(t *testing.T) { fx := newInitShellFixture(t, "systemd") if err := os.WriteFile(fx.params.UnitFile, []byte("PRIOR-UNIT-CONTENT"), 0o644); err != nil { t.Fatal(err) } if err := os.WriteFile(filepath.Join(fx.sysdDir, "rsmon-worker.service.enabled"), []byte(""), 0o644); err != nil { t.Fatal(err) } if err := os.WriteFile(filepath.Join(fx.sysdDir, "fail-restart"), []byte(""), 0o644); err != nil { t.Fatal(err) } out, err := fx.run(t) if err == nil { t.Fatal("activation succeeded despite restart failure") } if !strings.Contains(out, "start failure") { t.Fatalf("no start-failure classification:\n%s", out) } unit, err := os.ReadFile(fx.params.UnitFile) if err != nil { t.Fatal(err) } if string(unit) != "PRIOR-UNIT-CONTENT" { t.Fatalf("prior unit not restored: %q", unit) } log := fx.log(t, fx.sysctlLog) if !strings.Contains(log, "enable rsmon-worker.service") { t.Fatalf("prior enabled state not re-applied:\n%s", log) } if _, err := os.Stat(filepath.Join(fx.sysdDir, "rsmon-worker.service.enabled")); err != nil { t.Fatalf("unit not re-enabled after rollback: %v", err) } fx.assertNoLeftovers(t) } // TestActivateShellStaleLockBroken proves a leftover lock from a killed // run (dead pid) is broken automatically and the activation proceeds. func TestActivateShellStaleLockBroken(t *testing.T) { fx := newInitShellFixture(t, "systemd") lockDir := filepath.Join(fx.params.DataDir, ".rsmon-activate.lock") if err := os.MkdirAll(lockDir, 0o700); err != nil { t.Fatal(err) } if err := os.WriteFile(filepath.Join(lockDir, "pid"), []byte("999999\n"), 0o644); err != nil { t.Fatal(err) } out, err := fx.run(t) if err != nil { t.Fatalf("activation failed with a stale lock: %v\n%s", err, out) } if !strings.Contains(out, "rsmon-worker activated") { t.Fatalf("no activation success line:\n%s", out) } if _, err := os.Stat(lockDir); !os.IsNotExist(err) { t.Fatal("lock not released after activation") } } // TestActivateShellRecoveryFromInterruptedBackup proves the interrupted-run // recovery: a leftover backup marker from a killed activation is restored // before the fresh run, so a later validation failure still leaves the // recovered prior install in place. func TestActivateShellRecoveryFromInterruptedBackup(t *testing.T) { fx := newInitShellFixture(t, "none") backupDir := filepath.Join(fx.params.DataDir, ".rsmon-backup") if err := os.MkdirAll(backupDir, 0o700); err != nil { t.Fatal(err) } if err := os.WriteFile(filepath.Join(backupDir, "binary"), []byte("RECOVERED-BINARY"), 0o755); err != nil { t.Fatal(err) } if err := os.WriteFile(filepath.Join(backupDir, "worker.env"), []byte("RSMON_URL=https://rsmon.ru\nRSMON_TOKEN=test\n"), 0o600); err != nil { t.Fatal(err) } if err := os.WriteFile(filepath.Join(backupDir, ".marker"), []byte("0"), 0o600); err != nil { t.Fatal(err) } // Corrupt the stage so the fresh activation fails at validation (before // the fresh snapshot), proving the recovered files were already in place. if err := os.WriteFile(fx.params.Stage, []byte("not a script"), 0o644); err != nil { t.Fatal(err) } out, err := fx.run(t) if err == nil { t.Fatal("activation succeeded with a corrupt stage") } if !strings.Contains(out, "recovering interrupted activation") { t.Fatalf("recovery did not run:\n%s", out) } b, err := os.ReadFile(fx.params.Binary) if err != nil { t.Fatalf("recovered binary not restored: %v", err) } if string(b) != "RECOVERED-BINARY" { t.Fatalf("recovered binary content = %q", b) } if _, err := os.Stat(backupDir); !os.IsNotExist(err) { t.Fatal("backup not consumed by recovery") } }