package harness import ( "bytes" "context" "os" "path/filepath" "strings" "testing" "time" "golang.org/x/crypto/ssh" "rocketgit.ru/rsmon/worker/internal/installer" "rocketgit.ru/rsmon/worker/internal/sshinstall" ) // testWorkerToken is the operator-supplied worker token used by the // activation E2E tests. The fixture has no reachable control plane, but // the worker still binds its webapp and serves /healthz, which is what // activation verifies. const testWorkerToken = "e2e-activation-test-token" // activationOptions builds the work-package-4 source-install options for // a started fixture: full activation with a fixed test token, honoring // the RSMON_TEST_SOURCE_REPO/BRANCH overrides the staging tests use. func activationOptions(h *Harness, f Fixture) installer.SourceInstallOptions { opts := installer.SourceInstallOptions{ SSHOptions: installer.SSHOptions{ Host: "127.0.0.1", Port: h.Port(), User: f.UserOrDefault(), IdentityFile: testKeyPath(), KnownHostsFile: h.KnownHostsPath(), }, Activation: installer.ActivationOptions{ Activate: true, URL: "https://rsmon.ru", Token: testWorkerToken, }, } if repo := strings.TrimSpace(os.Getenv("RSMON_TEST_SOURCE_REPO")); repo != "" { opts.Repo = repo } if branch := strings.TrimSpace(os.Getenv("RSMON_TEST_SOURCE_BRANCH")); branch != "" { opts.Branch = branch } return opts } // fixtureByName returns the fixture entry for a fixture name. func fixtureByName(t *testing.T, name string) Fixture { t.Helper() for _, f := range Fixtures() { if f.Name == name { return f } } t.Fatalf("no fixture named %q", name) return Fixture{} } // startActivatedFixture starts a fixture container and runs a full // source install with activation against it, returning the harness, the // options (so a test can rerun with modifications), the result, and a // live SSH client. The caller owns client.Close and h.Stop (registered as // a test cleanup). func startActivatedFixture(t *testing.T, name string) (*Harness, installer.SourceInstallOptions, *installer.SourceInstallResult, *ssh.Client) { t.Helper() f := fixtureByName(t, name) h, err := New("activate-"+name, f) if err != nil { t.Fatal(err) } ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute) defer cancel() if err := h.Start(ctx); err != nil { t.Fatalf("start %s fixture: %v", name, err) } t.Cleanup(func() { if err := h.Stop(); err != nil { t.Errorf("stop %s fixture: %v", name, err) } }) opts := activationOptions(h, f) res, err := installer.SourceInstall(opts) if err != nil { t.Fatalf("activated source install on %s: %v", name, err) } client, err := h.Dial() if err != nil { t.Fatalf("dial %s after install: %v", name, err) } return h, opts, res, client } // countWorkerProcesses returns the number of running processes whose // command line is exactly the installed worker binary. The anchored -f // pattern matches both busybox and procps pgrep (busybox's -x matches the // full argv, so -x cannot be used portably). func countWorkerProcesses(t *testing.T, client *ssh.Client, binary string) string { t.Helper() out, err := RunCommand(client, "pgrep -f '^"+binary+"$' | wc -l") if err != nil { t.Fatalf("count worker processes: %v", err) } return strings.TrimSpace(string(out)) } // assertActivatedService verifies the full work-package-4 remote state: // the installed layout and permissions, the native service definition for // the detected init, exactly one running worker process, the pid file, // and a live /healthz (via the installed binary's own liveness // subcommand). func assertActivatedService(t *testing.T, client *ssh.Client, name string, res *installer.SourceInstallResult) { t.Helper() a := res.Activation if a == nil { t.Fatalf("%s: activation result missing", name) } out, err := RunCommand(client, "test -x "+shellQuote(a.Binary)+" && echo BIN_OK") if err != nil || !strings.Contains(string(out), "BIN_OK") { t.Fatalf("%s: installed binary not present/executable at %s: %q, %v", name, a.Binary, out, err) } envMode, err := RunCommand(client, "stat -c '%a' "+shellQuote(a.EnvFile)) if err != nil || strings.TrimSpace(string(envMode)) != "600" { t.Fatalf("%s: env file mode = %q, want 600 (%v)", name, envMode, err) } cfgMode, err := RunCommand(client, "stat -c '%a' "+shellQuote(a.ConfigDir)) if err != nil || strings.TrimSpace(string(cfgMode)) != "750" { t.Fatalf("%s: config dir mode = %q, want 750 (%v)", name, cfgMode, err) } if _, err := RunCommand(client, "test -d "+shellQuote(a.DataDir)+"/webapp && echo DATA_OK"); err != nil { t.Fatalf("%s: data dir not created at %s: %v", name, a.DataDir, err) } // The fixtures run no init system (sshd is PID 1), so activation // must use the embedded supervisor; the native service definition is // still installed for real hosts. if a.Supervisor != "none" { t.Fatalf("%s: supervisor = %q, want none in the fixture", name, a.Supervisor) } unitMode := "644" if res.Detection.InitSystem == sshinstall.InitOpenRC { unitMode = "755" } if a.UnitFile == "" { t.Fatalf("%s: no service definition installed", name) } if _, err := RunCommand(client, "test -f "+shellQuote(a.UnitFile)+" && echo UNIT_OK"); err != nil { t.Fatalf("%s: unit file missing at %s: %v", name, a.UnitFile, err) } gotUnitMode, err := RunCommand(client, "stat -c '%a' "+shellQuote(a.UnitFile)) if err != nil || strings.TrimSpace(string(gotUnitMode)) != unitMode { t.Fatalf("%s: unit mode = %q, want %s (%v)", name, gotUnitMode, unitMode, err) } // Exactly one worker process and a live /healthz. if got := countWorkerProcesses(t, client, a.Binary); got != "1" { t.Fatalf("%s: worker process count = %q, want 1", name, got) } if out, err := RunCommand(client, shellQuote(a.Binary)+" liveness"); err != nil || !strings.Contains(string(out), "ok") { t.Fatalf("%s: installed worker /healthz check failed: %q, %v", name, out, err) } pidOut, err := RunCommand(client, "cat "+shellQuote(a.DataDir)+"/worker.pid") if err != nil { t.Fatalf("%s: pid file missing: %v", name, err) } if _, err := RunCommand(client, "kill -0 "+strings.TrimSpace(string(pidOut))+" && echo PID_OK"); err != nil { t.Fatalf("%s: pid %s not alive: %v", name, pidOut, err) } } // assertNoActivationLeaks verifies a rerun left no backup dir, no lock, // no /tmp upload leftovers, and no extra worker processes. func assertNoActivationLeaks(t *testing.T, client *ssh.Client, name string, dataDir string) { t.Helper() out, err := RunCommand(client, "leak=0; [ -e "+shellQuote(dataDir)+"/.rsmon-backup ] && leak=1; "+ "[ -d "+shellQuote(dataDir)+"/.rsmon-activate.lock ] && leak=1; "+ "ls /tmp 2>/dev/null | grep -q '^rsmon-worker-act\\.' && leak=1; "+ "[ \"$leak\" -eq 1 ] && echo LEAK || echo CLEAN") if err != nil || strings.TrimSpace(string(out)) != "CLEAN" { t.Fatalf("%s: activation leftovers after rerun: %q, %v", name, out, err) } } // installedBinarySHA captures the SHA-256 of the installed worker binary // so failure tests can prove the prior binary is preserved. func installedBinarySHA(t *testing.T, client *ssh.Client, a *installer.ActivationResult) []byte { t.Helper() out, err := RunCommand(client, "sha256sum "+shellQuote(a.Binary)) if err != nil { t.Fatal(err) } return bytes.TrimSpace(out) } // TestSourceInstallActivationFixtures is the work-package-4 success and // idempotency acceptance test: on every fixture a full source install // ends with an atomically installed binary, a mode-0600 env file, the // data dir, a service definition for the detected init, and a running // verified worker. A rerun succeeds, keeps a single worker process, // rewrites the env deterministically, and leaks no temp or backup files. // // Opt-in: set RSMON_TEST_DOCKER=1 (see make test-ssh). func TestSourceInstallActivationFixtures(t *testing.T) { SkipUnlessEnabled(t) for _, f := range Fixtures() { f := f t.Run(f.Name, func(t *testing.T) { _, opts, res, client := startActivatedFixture(t, f.Name) defer client.Close() //nolint:errcheck assertActivatedService(t, client, f.Name, res) // Rerun: idempotent, single service, no leaks. res2, err := installer.SourceInstall(opts) if err != nil { t.Fatalf("%s: activated rerun: %v", f.Name, err) } if res2.Activation == nil || res2.ResolvedCommit != res.ResolvedCommit { t.Fatalf("%s: rerun lost activation/commit: %+v", f.Name, res2) } assertActivatedService(t, client, f.Name, res2) assertNoActivationLeaks(t, client, f.Name, res2.Activation.DataDir) envAfter, err := RunCommand(client, "cat "+shellQuote(res2.Activation.EnvFile)) if err != nil { t.Fatalf("%s: read env after rerun: %v", f.Name, err) } if !strings.Contains(string(envAfter), "RSMON_TOKEN="+testWorkerToken) { t.Fatalf("%s: env lost the worker token after rerun: %q", f.Name, envAfter) } }) } } // TestSourceInstallActivationFailureRollback is the work-package-4 // rollback acceptance test. After a successful activated install on the // Alpine fixture, each rerun is forced to fail at a different stage - // build (dirty checkout), activation (atomic swap sabotaged), start (the // worker process dies at boot), and health (/healthz unreachable) - and // every failure must leave the previous working install running with the // same binary, the same single process, and /healthz answering. // // Opt-in: set RSMON_TEST_DOCKER=1 (see make test-ssh). func TestSourceInstallActivationFailureRollback(t *testing.T) { SkipUnlessEnabled(t) const fixture = "alpine" _, opts, res, client := startActivatedFixture(t, fixture) defer client.Close() //nolint:errcheck a := res.Activation assertActivatedService(t, client, fixture, res) // 1. Build failure preserves the prior install. // A dirty tracked tree makes the rerun's checkout refuse before // the build, so neither staging nor the service changes. Appending // a marker is deterministic regardless of the cloned history (a // `git checkout master~1 -- ` would depend on whether that // commit differs from the resolved one). if _, err := RunCommand(client, "printf '\\n# rsmon-worker dirty-tree marker\\n' >> "+shellQuote(res.Plan.BuildDir)+"/Makefile"); err != nil { t.Fatalf("dirty the working tree: %v", err) } before := installedBinarySHA(t, client, a) _, err := installer.SourceInstall(opts) if err == nil || !strings.Contains(err.Error(), "check out branch") { t.Fatalf("rerun on dirty tree err = %v, want checkout failure", err) } assertActivatedService(t, client, fixture, res) if !bytes.Equal(before, installedBinarySHA(t, client, a)) { t.Fatalf("installed binary changed after build-failure rerun") } // Restore the tree so later reruns can check out again. if _, err := RunCommand(client, "git -C "+shellQuote(res.Plan.BuildDir)+" checkout -q HEAD -- Makefile"); err != nil { t.Fatalf("restore working tree: %v", err) } // 2. Activation failure preserves the prior install. // Pre-creating the atomic-swap temp path as a directory makes the // binary install step fail mid-activation; the rollback trap must // restore the previous binary/env and restart the prior service. if _, err := RunCommand(client, "mkdir -p /usr/local/bin/rsmon-worker.new"); err != nil { t.Fatalf("sabotage the atomic binary swap: %v", err) } _, err = installer.SourceInstall(opts) if err == nil || !strings.Contains(err.Error(), "activate worker service") { t.Fatalf("activation-failure rerun err = %v, want activate failure", err) } assertActivatedService(t, client, fixture, res) if !bytes.Equal(before, installedBinarySHA(t, client, a)) { t.Fatalf("installed binary changed after activation-failure rerun") } assertNoActivationLeaks(t, client, fixture, a.DataDir) if _, err := RunCommand(client, "rm -rf /usr/local/bin/rsmon-worker.new"); err != nil { t.Fatalf("clean the swap sabotage: %v", err) } // 3. Start failure preserves the prior install. // WORKER_CLUSTER_ENABLED=true without WORKER_LOGIN/PASSWORD makes // the new worker exit at boot (cluster init is fatal), so the // activation supervisor detects the process died and rolls back. envPath := filepath.Join(t.TempDir(), "worker.env") startEnv := "RSMON_URL=https://rsmon.ru\nRSMON_TOKEN=" + testWorkerToken + "\nWORKER_CLUSTER_ENABLED=true\n" if err := os.WriteFile(envPath, []byte(startEnv), 0o600); err != nil { t.Fatal(err) } startOpts := opts startOpts.Activation.EnvFile = envPath _, err = installer.SourceInstall(startOpts) if err == nil || !strings.Contains(err.Error(), "start failure") { t.Fatalf("start-failure rerun err = %v, want start failure", err) } assertActivatedService(t, client, fixture, res) if !bytes.Equal(before, installedBinarySHA(t, client, a)) { t.Fatalf("installed binary changed after start-failure rerun") } assertNoActivationLeaks(t, client, fixture, a.DataDir) // 4. Health failure preserves the prior install. // WORKER_HOST=255.255.255.255 makes the new worker bind fail and // /healthz unreachable; the process stays up but the health gate // fails and the rollback restores the prior loopback deployment. healthOpts := opts healthOpts.Activation.Host = "255.255.255.255" _, err = installer.SourceInstall(healthOpts) if err == nil || !strings.Contains(err.Error(), "health failure") { t.Fatalf("health-failure rerun err = %v, want health failure", err) } assertActivatedService(t, client, fixture, res) if !bytes.Equal(before, installedBinarySHA(t, client, a)) { t.Fatalf("installed binary changed after health-failure rerun") } assertNoActivationLeaks(t, client, fixture, a.DataDir) } // TestSourceInstallActivationNoStart installs the full layout (binary, // env, data dir, service definition) without starting the worker, and // asserts nothing runs afterward. Reruns stay idempotent. // // Opt-in: set RSMON_TEST_DOCKER=1 (see make test-ssh). func TestSourceInstallActivationNoStart(t *testing.T) { SkipUnlessEnabled(t) const fixture = "alpine" f := fixtureByName(t, fixture) h, err := New("nostart-"+fixture, f) if err != nil { t.Fatal(err) } ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute) defer cancel() if err := h.Start(ctx); err != nil { t.Fatalf("start %s fixture: %v", fixture, err) } t.Cleanup(func() { if err := h.Stop(); err != nil { t.Errorf("stop: %v", err) } }) opts := activationOptions(h, f) opts.Activation.NoStart = true res, err := installer.SourceInstall(opts) if err != nil { t.Fatalf("no-start install: %v", err) } client, err := h.Dial() if err != nil { t.Fatalf("dial: %v", err) } defer client.Close() //nolint:errcheck a := res.Activation if a == nil || a.Started { t.Fatalf("no-start activation result = %+v", a) } if out, err := RunCommand(client, "test -x "+shellQuote(a.Binary)+" && test -f "+shellQuote(a.EnvFile)+" && echo INSTALLED"); err != nil || !strings.Contains(string(out), "INSTALLED") { t.Fatalf("no-start did not install the layout: %q, %v", out, err) } if got := countWorkerProcesses(t, client, a.Binary); got != "0" { t.Fatalf("no-start left a running worker: %q", got) } // A rerun is idempotent and still starts nothing. if _, err := installer.SourceInstall(opts); err != nil { t.Fatalf("no-start rerun: %v", err) } if got := countWorkerProcesses(t, client, a.Binary); got != "0" { t.Fatalf("no-start rerun left a running worker: %q", got) } assertNoActivationLeaks(t, client, fixture, a.DataDir) }