worker/internal/installer/harness/integration_activation_test.go

package harness

import (
	"bytes"
	"context"
	"os"
	"path/filepath"
	"strings"
	"testing"
	"time"

	"golang.org/x/crypto/ssh"

	"rocketgit.ru/rsmon/worker/internal/installer"
	"rocketgit.ru/rsmon/worker/internal/sshinstall"
)

// testWorkerToken is the operator-supplied worker token used by the
// activation E2E tests. The fixture has no reachable control plane, but
// the worker still binds its webapp and serves /healthz, which is what
// activation verifies.
const testWorkerToken = "e2e-activation-test-token"

// activationOptions builds the work-package-4 source-install options for
// a started fixture: full activation with a fixed test token, honoring
// the RSMON_TEST_SOURCE_REPO/BRANCH overrides the staging tests use.
func activationOptions(h *Harness, f Fixture) installer.SourceInstallOptions {
	opts := installer.SourceInstallOptions{
		SSHOptions: installer.SSHOptions{
			Host:           "127.0.0.1",
			Port:           h.Port(),
			User:           f.UserOrDefault(),
			IdentityFile:   testKeyPath(),
			KnownHostsFile: h.KnownHostsPath(),
		},
		Activation: installer.ActivationOptions{
			Activate: true,
			URL:      "https://rsmon.ru",
			Token:    testWorkerToken,
		},
	}
	if repo := strings.TrimSpace(os.Getenv("RSMON_TEST_SOURCE_REPO")); repo != "" {
		opts.Repo = repo
	}
	if branch := strings.TrimSpace(os.Getenv("RSMON_TEST_SOURCE_BRANCH")); branch != "" {
		opts.Branch = branch
	}
	return opts
}

// fixtureByName returns the fixture entry for a fixture name.
func fixtureByName(t *testing.T, name string) Fixture {
	t.Helper()
	for _, f := range Fixtures() {
		if f.Name == name {
			return f
		}
	}
	t.Fatalf("no fixture named %q", name)
	return Fixture{}
}

// startActivatedFixture starts a fixture container and runs a full
// source install with activation against it, returning the harness, the
// options (so a test can rerun with modifications), the result, and a
// live SSH client. The caller owns client.Close and h.Stop (registered as
// a test cleanup).
func startActivatedFixture(t *testing.T, name string) (*Harness, installer.SourceInstallOptions, *installer.SourceInstallResult, *ssh.Client) {
	t.Helper()
	f := fixtureByName(t, name)
	h, err := New("activate-"+name, f)
	if err != nil {
		t.Fatal(err)
	}
	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute)
	defer cancel()
	if err := h.Start(ctx); err != nil {
		t.Fatalf("start %s fixture: %v", name, err)
	}
	t.Cleanup(func() {
		if err := h.Stop(); err != nil {
			t.Errorf("stop %s fixture: %v", name, err)
		}
	})
	opts := activationOptions(h, f)
	res, err := installer.SourceInstall(opts)
	if err != nil {
		t.Fatalf("activated source install on %s: %v", name, err)
	}
	client, err := h.Dial()
	if err != nil {
		t.Fatalf("dial %s after install: %v", name, err)
	}
	return h, opts, res, client
}

// countWorkerProcesses returns the number of running processes whose
// command line is exactly the installed worker binary. The anchored -f
// pattern matches both busybox and procps pgrep (busybox's -x matches the
// full argv, so -x cannot be used portably).
func countWorkerProcesses(t *testing.T, client *ssh.Client, binary string) string {
	t.Helper()
	out, err := RunCommand(client, "pgrep -f '^"+binary+"$' | wc -l")
	if err != nil {
		t.Fatalf("count worker processes: %v", err)
	}
	return strings.TrimSpace(string(out))
}

// assertActivatedService verifies the full work-package-4 remote state:
// the installed layout and permissions, the native service definition for
// the detected init, exactly one running worker process, the pid file,
// and a live /healthz (via the installed binary's own liveness
// subcommand).
func assertActivatedService(t *testing.T, client *ssh.Client, name string, res *installer.SourceInstallResult) {
	t.Helper()
	a := res.Activation
	if a == nil {
		t.Fatalf("%s: activation result missing", name)
	}

	out, err := RunCommand(client, "test -x "+shellQuote(a.Binary)+" && echo BIN_OK")
	if err != nil || !strings.Contains(string(out), "BIN_OK") {
		t.Fatalf("%s: installed binary not present/executable at %s: %q, %v", name, a.Binary, out, err)
	}

	envMode, err := RunCommand(client, "stat -c '%a' "+shellQuote(a.EnvFile))
	if err != nil || strings.TrimSpace(string(envMode)) != "600" {
		t.Fatalf("%s: env file mode = %q, want 600 (%v)", name, envMode, err)
	}
	cfgMode, err := RunCommand(client, "stat -c '%a' "+shellQuote(a.ConfigDir))
	if err != nil || strings.TrimSpace(string(cfgMode)) != "750" {
		t.Fatalf("%s: config dir mode = %q, want 750 (%v)", name, cfgMode, err)
	}
	if _, err := RunCommand(client, "test -d "+shellQuote(a.DataDir)+"/webapp && echo DATA_OK"); err != nil {
		t.Fatalf("%s: data dir not created at %s: %v", name, a.DataDir, err)
	}

	// The fixtures run no init system (sshd is PID 1), so activation
	// must use the embedded supervisor; the native service definition is
	// still installed for real hosts.
	if a.Supervisor != "none" {
		t.Fatalf("%s: supervisor = %q, want none in the fixture", name, a.Supervisor)
	}
	unitMode := "644"
	if res.Detection.InitSystem == sshinstall.InitOpenRC {
		unitMode = "755"
	}
	if a.UnitFile == "" {
		t.Fatalf("%s: no service definition installed", name)
	}
	if _, err := RunCommand(client, "test -f "+shellQuote(a.UnitFile)+" && echo UNIT_OK"); err != nil {
		t.Fatalf("%s: unit file missing at %s: %v", name, a.UnitFile, err)
	}
	gotUnitMode, err := RunCommand(client, "stat -c '%a' "+shellQuote(a.UnitFile))
	if err != nil || strings.TrimSpace(string(gotUnitMode)) != unitMode {
		t.Fatalf("%s: unit mode = %q, want %s (%v)", name, gotUnitMode, unitMode, err)
	}

	// Exactly one worker process and a live /healthz.
	if got := countWorkerProcesses(t, client, a.Binary); got != "1" {
		t.Fatalf("%s: worker process count = %q, want 1", name, got)
	}
	if out, err := RunCommand(client, shellQuote(a.Binary)+" liveness"); err != nil || !strings.Contains(string(out), "ok") {
		t.Fatalf("%s: installed worker /healthz check failed: %q, %v", name, out, err)
	}
	pidOut, err := RunCommand(client, "cat "+shellQuote(a.DataDir)+"/worker.pid")
	if err != nil {
		t.Fatalf("%s: pid file missing: %v", name, err)
	}
	if _, err := RunCommand(client, "kill -0 "+strings.TrimSpace(string(pidOut))+" && echo PID_OK"); err != nil {
		t.Fatalf("%s: pid %s not alive: %v", name, pidOut, err)
	}
}

// assertNoActivationLeaks verifies a rerun left no backup dir, no lock,
// no /tmp upload leftovers, and no extra worker processes.
func assertNoActivationLeaks(t *testing.T, client *ssh.Client, name string, dataDir string) {
	t.Helper()
	out, err := RunCommand(client,
		"leak=0; [ -e "+shellQuote(dataDir)+"/.rsmon-backup ] && leak=1; "+
			"[ -d "+shellQuote(dataDir)+"/.rsmon-activate.lock ] && leak=1; "+
			"ls /tmp 2>/dev/null | grep -q '^rsmon-worker-act\\.' && leak=1; "+
			"[ \"$leak\" -eq 1 ] && echo LEAK || echo CLEAN")
	if err != nil || strings.TrimSpace(string(out)) != "CLEAN" {
		t.Fatalf("%s: activation leftovers after rerun: %q, %v", name, out, err)
	}
}

// installedBinarySHA captures the SHA-256 of the installed worker binary
// so failure tests can prove the prior binary is preserved.
func installedBinarySHA(t *testing.T, client *ssh.Client, a *installer.ActivationResult) []byte {
	t.Helper()
	out, err := RunCommand(client, "sha256sum "+shellQuote(a.Binary))
	if err != nil {
		t.Fatal(err)
	}
	return bytes.TrimSpace(out)
}

// TestSourceInstallActivationFixtures is the work-package-4 success and
// idempotency acceptance test: on every fixture a full source install
// ends with an atomically installed binary, a mode-0600 env file, the
// data dir, a service definition for the detected init, and a running
// verified worker. A rerun succeeds, keeps a single worker process,
// rewrites the env deterministically, and leaks no temp or backup files.
//
// Opt-in: set RSMON_TEST_DOCKER=1 (see make test-ssh).
func TestSourceInstallActivationFixtures(t *testing.T) {
	SkipUnlessEnabled(t)

	for _, f := range Fixtures() {
		f := f
		t.Run(f.Name, func(t *testing.T) {
			_, opts, res, client := startActivatedFixture(t, f.Name)
			defer client.Close() //nolint:errcheck
			assertActivatedService(t, client, f.Name, res)

			// Rerun: idempotent, single service, no leaks.
			res2, err := installer.SourceInstall(opts)
			if err != nil {
				t.Fatalf("%s: activated rerun: %v", f.Name, err)
			}
			if res2.Activation == nil || res2.ResolvedCommit != res.ResolvedCommit {
				t.Fatalf("%s: rerun lost activation/commit: %+v", f.Name, res2)
			}
			assertActivatedService(t, client, f.Name, res2)
			assertNoActivationLeaks(t, client, f.Name, res2.Activation.DataDir)

			envAfter, err := RunCommand(client, "cat "+shellQuote(res2.Activation.EnvFile))
			if err != nil {
				t.Fatalf("%s: read env after rerun: %v", f.Name, err)
			}
			if !strings.Contains(string(envAfter), "RSMON_TOKEN="+testWorkerToken) {
				t.Fatalf("%s: env lost the worker token after rerun: %q", f.Name, envAfter)
			}
		})
	}
}

// TestSourceInstallActivationFailureRollback is the work-package-4
// rollback acceptance test. After a successful activated install on the
// Alpine fixture, each rerun is forced to fail at a different stage -
// build (dirty checkout), activation (atomic swap sabotaged), start (the
// worker process dies at boot), and health (/healthz unreachable) - and
// every failure must leave the previous working install running with the
// same binary, the same single process, and /healthz answering.
//
// Opt-in: set RSMON_TEST_DOCKER=1 (see make test-ssh).
func TestSourceInstallActivationFailureRollback(t *testing.T) {
	SkipUnlessEnabled(t)

	const fixture = "alpine"
	_, opts, res, client := startActivatedFixture(t, fixture)
	defer client.Close() //nolint:errcheck
	a := res.Activation
	assertActivatedService(t, client, fixture, res)

	// 1. Build failure preserves the prior install.
	//    A dirty tracked tree makes the rerun's checkout refuse before
	//    the build, so neither staging nor the service changes.
	if _, err := RunCommand(client, "git -C "+shellQuote(res.Plan.BuildDir)+" checkout -q master~1 -- Makefile"); err != nil {
		t.Fatalf("dirty the working tree: %v", err)
	}
	before := installedBinarySHA(t, client, a)
	_, err := installer.SourceInstall(opts)
	if err == nil || !strings.Contains(err.Error(), "check out branch") {
		t.Fatalf("rerun on dirty tree err = %v, want checkout failure", err)
	}
	assertActivatedService(t, client, fixture, res)
	if !bytes.Equal(before, installedBinarySHA(t, client, a)) {
		t.Fatalf("installed binary changed after build-failure rerun")
	}
	// Restore the tree so later reruns can check out again.
	if _, err := RunCommand(client, "git -C "+shellQuote(res.Plan.BuildDir)+" checkout -q HEAD -- Makefile"); err != nil {
		t.Fatalf("restore working tree: %v", err)
	}

	// 2. Activation failure preserves the prior install.
	//    Pre-creating the atomic-swap temp path as a directory makes the
	//    binary install step fail mid-activation; the rollback trap must
	//    restore the previous binary/env and restart the prior service.
	if _, err := RunCommand(client, "mkdir -p /usr/local/bin/rsmon-worker.new"); err != nil {
		t.Fatalf("sabotage the atomic binary swap: %v", err)
	}
	_, err = installer.SourceInstall(opts)
	if err == nil || !strings.Contains(err.Error(), "activate worker service") {
		t.Fatalf("activation-failure rerun err = %v, want activate failure", err)
	}
	assertActivatedService(t, client, fixture, res)
	if !bytes.Equal(before, installedBinarySHA(t, client, a)) {
		t.Fatalf("installed binary changed after activation-failure rerun")
	}
	assertNoActivationLeaks(t, client, fixture, a.DataDir)
	if _, err := RunCommand(client, "rm -rf /usr/local/bin/rsmon-worker.new"); err != nil {
		t.Fatalf("clean the swap sabotage: %v", err)
	}

	// 3. Start failure preserves the prior install.
	//    WORKER_CLUSTER_ENABLED=true without WORKER_LOGIN/PASSWORD makes
	//    the new worker exit at boot (cluster init is fatal), so the
	//    activation supervisor detects the process died and rolls back.
	envPath := filepath.Join(t.TempDir(), "worker.env")
	startEnv := "RSMON_URL=https://rsmon.ru\nRSMON_TOKEN=" + testWorkerToken + "\nWORKER_CLUSTER_ENABLED=true\n"
	if err := os.WriteFile(envPath, []byte(startEnv), 0o600); err != nil {
		t.Fatal(err)
	}
	startOpts := opts
	startOpts.Activation.EnvFile = envPath
	_, err = installer.SourceInstall(startOpts)
	if err == nil || !strings.Contains(err.Error(), "start failure") {
		t.Fatalf("start-failure rerun err = %v, want start failure", err)
	}
	assertActivatedService(t, client, fixture, res)
	if !bytes.Equal(before, installedBinarySHA(t, client, a)) {
		t.Fatalf("installed binary changed after start-failure rerun")
	}
	assertNoActivationLeaks(t, client, fixture, a.DataDir)

	// 4. Health failure preserves the prior install.
	//    WORKER_HOST=255.255.255.255 makes the new worker bind fail and
	//    /healthz unreachable; the process stays up but the health gate
	//    fails and the rollback restores the prior loopback deployment.
	healthOpts := opts
	healthOpts.Activation.Host = "255.255.255.255"
	_, err = installer.SourceInstall(healthOpts)
	if err == nil || !strings.Contains(err.Error(), "health failure") {
		t.Fatalf("health-failure rerun err = %v, want health failure", err)
	}
	assertActivatedService(t, client, fixture, res)
	if !bytes.Equal(before, installedBinarySHA(t, client, a)) {
		t.Fatalf("installed binary changed after health-failure rerun")
	}
	assertNoActivationLeaks(t, client, fixture, a.DataDir)
}

// TestSourceInstallActivationNoStart installs the full layout (binary,
// env, data dir, service definition) without starting the worker, and
// asserts nothing runs afterward. Reruns stay idempotent.
//
// Opt-in: set RSMON_TEST_DOCKER=1 (see make test-ssh).
func TestSourceInstallActivationNoStart(t *testing.T) {
	SkipUnlessEnabled(t)

	const fixture = "alpine"
	f := fixtureByName(t, fixture)
	h, err := New("nostart-"+fixture, f)
	if err != nil {
		t.Fatal(err)
	}
	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute)
	defer cancel()
	if err := h.Start(ctx); err != nil {
		t.Fatalf("start %s fixture: %v", fixture, err)
	}
	t.Cleanup(func() {
		if err := h.Stop(); err != nil {
			t.Errorf("stop: %v", err)
		}
	})
	opts := activationOptions(h, f)
	opts.Activation.NoStart = true
	res, err := installer.SourceInstall(opts)
	if err != nil {
		t.Fatalf("no-start install: %v", err)
	}
	client, err := h.Dial()
	if err != nil {
		t.Fatalf("dial: %v", err)
	}
	defer client.Close() //nolint:errcheck
	a := res.Activation
	if a == nil || a.Started {
		t.Fatalf("no-start activation result = %+v", a)
	}
	if out, err := RunCommand(client, "test -x "+shellQuote(a.Binary)+" && test -f "+shellQuote(a.EnvFile)+" && echo INSTALLED"); err != nil || !strings.Contains(string(out), "INSTALLED") {
		t.Fatalf("no-start did not install the layout: %q, %v", out, err)
	}
	if got := countWorkerProcesses(t, client, a.Binary); got != "0" {
		t.Fatalf("no-start left a running worker: %q", got)
	}
	// A rerun is idempotent and still starts nothing.
	if _, err := installer.SourceInstall(opts); err != nil {
		t.Fatalf("no-start rerun: %v", err)
	}
	if got := countWorkerProcesses(t, client, a.Binary); got != "0" {
		t.Fatalf("no-start rerun left a running worker: %q", got)
	}
	assertNoActivationLeaks(t, client, fixture, a.DataDir)
}