worker/internal/installer/harness/integration_test.go

package harness

import (
	"bytes"
	"context"
	"crypto/ed25519"
	"crypto/rand"
	"fmt"
	"os"
	"path/filepath"
	"strings"
	"testing"
	"time"

	"golang.org/x/crypto/ssh"
	"golang.org/x/crypto/ssh/knownhosts"

	"rocketgit.ru/rsmon/worker/internal/installer"
	"rocketgit.ru/rsmon/worker/internal/sshinstall"
)

// TestHarnessFixtures is the work-package-1 acceptance test: each distro
// fixture starts a real OpenSSH container, the harness waits for real
// network readiness, and the installer's Go SSH client connects, runs
// commands, and tears the environment down.
//
// Opt-in: set RSMON_TEST_DOCKER=1 (see make test-ssh).
func TestHarnessFixtures(t *testing.T) {
	SkipUnlessEnabled(t)

	for _, f := range Fixtures() {
		f := f
		t.Run(f.Name, func(t *testing.T) {
			h, err := New("fixture-"+f.Name, f)
			if err != nil {
				t.Fatal(err)
			}
			ctx, cancel := context.WithTimeout(context.Background(), 20*time.Minute)
			defer cancel()
			if err := h.Start(ctx); err != nil {
				t.Fatalf("start %s fixture: %v", f.Name, err)
			}
			t.Cleanup(func() {
				if err := h.Stop(); err != nil {
					t.Errorf("stop %s fixture: %v", f.Name, err)
				}
			})

			client, err := h.Dial()
			if err != nil {
				t.Fatalf("dial %s fixture: %v", f.Name, err)
			}
			defer client.Close() //nolint:errcheck

			// 1. SSH command execution is real: round-trip a nonce.
			nonce := fmt.Sprintf("RSMON_SSH_OK_%d", time.Now().UnixNano())
			out, err := RunCommand(client, "printf '%s' "+shellQuote(nonce))
			if err != nil {
				t.Fatalf("ssh round trip: %v", err)
			}
			if strings.TrimSpace(string(out)) != nonce {
				t.Fatalf("ssh round trip = %q, want %q", out, nonce)
			}

			// 2. The clean target has no Go toolchain and no worker source.
			out, err = RunCommand(client, "command -v go || true; test ! -e /usr/local/go && echo NO_GO; test ! -e /opt/rsmon-worker-src && echo NO_SOURCE")
			if err != nil {
				t.Fatalf("clean-state probe: %v", err)
			}
			clean := string(out)
			if strings.Contains(clean, "/go") && !strings.Contains(clean, "NO_GO") {
				t.Fatalf("fixture unexpectedly has Go installed: %q", clean)
			}
			if !strings.Contains(clean, "NO_SOURCE") {
				t.Fatalf("fixture unexpectedly has worker source: %q", clean)
			}

			// 3. Distro detection over the real session matches the fixture.
			out, err = RunCommand(client, "cat /etc/os-release")
			if err != nil {
				t.Fatalf("read os-release: %v", err)
			}
			d := sshinstall.Detect(string(out), makeProber(client))
			if d.Distro != f.Distro {
				t.Fatalf("detected distro = %q, want %q (%s)", d.Distro, f.Distro, d.Summarize())
			}
			if d.PackageManager != f.Pkg {
				t.Fatalf("detected package manager = %q, want %q", d.PackageManager, f.Pkg)
			}
			if d.InitSystem != f.Init {
				t.Fatalf("detected init = %q, want %q", d.InitSystem, f.Init)
			}
			t.Logf("%s: %s", f.Name, d.Summarize())

			// 4. A full source plan resolves for the detected host,
			// including the pinned Go toolchain for its real arch.
			out, err = RunCommand(client, "uname -m")
			if err != nil {
				t.Fatalf("uname -m: %v", err)
			}
			goarch, err := sshinstall.GoArch(strings.TrimSpace(string(out)))
			if err != nil {
				t.Fatalf("GoArch(%q): %v", out, err)
			}
			plan, err := sshinstall.PlanSource(d, sshinstall.SourceOptions{UnameM: strings.TrimSpace(string(out))})
			if err != nil {
				t.Fatalf("PlanSource: %v", err)
			}
			if plan.Toolchain.Arch != "linux-"+goarch {
				t.Fatalf("plan toolchain %q does not match detected arch %q", plan.Toolchain.Arch, goarch)
			}
			if len(plan.Packages) == 0 || plan.Repo == "" {
				t.Fatalf("incomplete plan: %+v", plan)
			}
			steps := plan.Steps()
			if len(steps) != 6 {
				t.Fatalf("plan steps = %d, want 6", len(steps))
			}
		})
	}
}

// TestHarnessHostKeyMismatch verifies the security gate: a dial against
// a known_hosts entry carrying a different host key must fail before any
// command can run.
func TestHarnessHostKeyMismatch(t *testing.T) {
	SkipUnlessEnabled(t)

	f := Fixtures()[0] // alpine is the smallest fixture
	h, err := New("hostkey-"+f.Name, f)
	if err != nil {
		t.Fatal(err)
	}
	ctx, cancel := context.WithTimeout(context.Background(), 20*time.Minute)
	defer cancel()
	if err := h.Start(ctx); err != nil {
		t.Fatalf("start fixture: %v", err)
	}
	t.Cleanup(func() {
		if err := h.Stop(); err != nil {
			t.Errorf("stop fixture: %v", err)
		}
	})

	// Build a known_hosts entry with a different (freshly generated) key.
	_, priv, err := ed25519.GenerateKey(rand.Reader)
	if err != nil {
		t.Fatal(err)
	}
	signer, err := ssh.NewSignerFromKey(priv)
	if err != nil {
		t.Fatal(err)
	}
	wrongFile := filepath.Join(t.TempDir(), "known_hosts")
	line := knownhosts.Line([]string{h.Addr()}, signer.PublicKey())
	if err := os.WriteFile(wrongFile, []byte(line+"\n"), 0o600); err != nil {
		t.Fatal(err)
	}

	keyBytes, err := os.ReadFile(testKeyPath())
	if err != nil {
		t.Fatal(err)
	}
	keySigner, err := ssh.ParsePrivateKey(keyBytes)
	if err != nil {
		t.Fatal(err)
	}
	callback, err := knownhosts.New(wrongFile)
	if err != nil {
		t.Fatal(err)
	}
	config := &ssh.ClientConfig{
		User:            f.UserOrDefault(),
		Auth:            []ssh.AuthMethod{ssh.PublicKeys(keySigner)},
		HostKeyCallback: callback,
		Timeout:         15 * time.Second,
	}
	client, err := ssh.Dial("tcp", h.Addr(), config)
	if err == nil {
		client.Close() //nolint:errcheck
		t.Fatal("dial with a mismatched host key succeeded")
	}
	if !strings.Contains(err.Error(), "knownhosts") && !strings.Contains(err.Error(), "key") {
		t.Fatalf("host-key mismatch error = %v, want a key/host verification failure", err)
	}
}

// TestHarnessTeardown verifies Stop reliably removes the container, the
// dedicated network, the per-instance fixture image tag (never a shared
// base image), and the temp known_hosts directory.
func TestHarnessTeardown(t *testing.T) {
	SkipUnlessEnabled(t)

	f := Fixtures()[0]
	h, err := New("teardown-"+f.Name, f)
	if err != nil {
		t.Fatal(err)
	}
	ctx, cancel := context.WithTimeout(context.Background(), 20*time.Minute)
	defer cancel()
	if err := h.Start(ctx); err != nil {
		t.Fatalf("start fixture: %v", err)
	}
	if h.ContainerID() == "" {
		t.Fatal("container id empty after start")
	}
	if h.KnownHostsPath() == "" {
		t.Fatal("known_hosts not created after start")
	}
	knownHostsDir := filepath.Dir(h.KnownHostsPath())
	if err := h.Stop(); err != nil {
		t.Fatalf("stop: %v", err)
	}
	if _, err := h.DockerCmd(ctx, "inspect", h.ContainerName()); err == nil {
		t.Fatal("container still present after Stop")
	}
	if _, err := h.DockerCmd(ctx, "network", "inspect", h.NetworkName()); err == nil {
		t.Fatal("network still present after Stop")
	}
	if _, err := h.DockerCmd(ctx, "image", "inspect", h.ImageTag()); err == nil {
		t.Fatalf("fixture image tag %q still present after Stop", h.ImageTag())
	}
	if _, err := os.Stat(knownHostsDir); !os.IsNotExist(err) {
		t.Fatalf("known_hosts temp dir %q still present after Stop: %v", knownHostsDir, err)
	}
	if h.KnownHostsPath() != "" {
		t.Fatalf("KnownHostsPath = %q after Stop, want empty", h.KnownHostsPath())
	}
	// Stop is idempotent.
	if err := h.Stop(); err != nil {
		t.Fatalf("second stop: %v", err)
	}
}

// TestHarnessHostKeyStableAcrossDial ensures the host key captured at
// readiness is the one verified on every later dial, so a successful
// Dial is proof of verified, real SSH transport.
func TestHarnessHostKeyStable(t *testing.T) {
	SkipUnlessEnabled(t)

	f := Fixtures()[0]
	h, err := New("key-"+f.Name, f)
	if err != nil {
		t.Fatal(err)
	}
	ctx, cancel := context.WithTimeout(context.Background(), 20*time.Minute)
	defer cancel()
	if err := h.Start(ctx); err != nil {
		t.Fatalf("start fixture: %v", err)
	}
	t.Cleanup(func() {
		if err := h.Stop(); err != nil {
			t.Errorf("stop fixture: %v", err)
		}
	})
	client, err := h.Dial()
	if err != nil {
		t.Fatalf("dial: %v", err)
	}
	client.Close() //nolint:errcheck
	client, err = h.Dial()
	if err != nil {
		t.Fatalf("second dial: %v", err)
	}
	defer client.Close() //nolint:errcheck
	if out, err := RunCommand(client, "echo VERIFIED"); err != nil || strings.TrimSpace(string(out)) != "VERIFIED" {
		t.Fatalf("verified session command = %q, %v", out, err)
	}
}

// TestSourceInstallFixtures is the work-package-3 acceptance test: each
// distro fixture starts clean (no Go, no worker source) and the real
// installer executes the full source flow over SSH - prerequisite
// install, verified Go toolchain download/extraction, clone/update of the
// public repository, resolved branch/commit record, and a build to a
// staging path. The running service and its config are deliberately not
// installed (that is work package 4). A rerun exercises idempotency.
//
// Opt-in: set RSMON_TEST_DOCKER=1 (see make test-ssh).
func TestSourceInstallFixtures(t *testing.T) {
	SkipUnlessEnabled(t)

	for _, f := range Fixtures() {
		f := f
		t.Run(f.Name, func(t *testing.T) {
			h, err := New("source-"+f.Name, f)
			if err != nil {
				t.Fatal(err)
			}
			ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute)
			defer cancel()
			if err := h.Start(ctx); err != nil {
				t.Fatalf("start %s fixture: %v", f.Name, err)
			}
			t.Cleanup(func() {
				if err := h.Stop(); err != nil {
					t.Errorf("stop %s fixture: %v", f.Name, err)
				}
			})

			// Clean state: no Go toolchain, no source, no leftover
			// toolchain temp dirs.
			client, err := h.Dial()
			if err != nil {
				t.Fatalf("dial %s fixture: %v", f.Name, err)
			}
			probe, err := RunCommand(client, "command -v go || true; test ! -e /usr/local/go && echo NO_GO; test ! -e /opt/rsmon-worker-src && echo NO_SOURCE; ls /tmp | grep -q rsmon-toolchain && echo LEAK; ls /usr/local 2>/dev/null | grep -qE 'go-staging|go-backup' && echo LEAK; echo DONE")
			if err != nil {
				t.Fatalf("clean-state probe: %v", err)
			}
			clean := string(probe)
			if !strings.Contains(clean, "NO_GO") || !strings.Contains(clean, "NO_SOURCE") {
				t.Fatalf("fixture is not clean: %q", clean)
			}
			if strings.Contains(clean, "LEAK") {
				t.Fatalf("fixture has leftover toolchain temp dirs: %q", clean)
			}
			client.Close() //nolint:errcheck

			opts := installer.SourceInstallOptions{
				SSHOptions: installer.SSHOptions{
					Host:           "127.0.0.1",
					Port:           h.Port(),
					User:           f.UserOrDefault(),
					IdentityFile:   testKeyPath(),
					KnownHostsFile: h.KnownHostsPath(),
				},
				// Staging-only: service activation (work package 4) is
				// exercised by the dedicated activation tests.
				Activation: installer.ActivationOptions{Activate: false},
			}
			if repo := strings.TrimSpace(os.Getenv("RSMON_TEST_SOURCE_REPO")); repo != "" {
				opts.Repo = repo
			}
			if branch := strings.TrimSpace(os.Getenv("RSMON_TEST_SOURCE_BRANCH")); branch != "" {
				opts.Branch = branch
			}

			res, err := installer.SourceInstall(opts)
			if err != nil {
				t.Fatalf("source install on %s: %v", f.Name, err)
			}
			t.Logf("%s: %s -> branch=%s commit=%s staged=%s", f.Name, res.Detection.Summarize(),
				res.ResolvedBranch, res.ResolvedCommit, res.StageBinary)

			if res.Detection.Distro != f.Distro || res.Detection.PackageManager != f.Pkg || res.Detection.InitSystem != f.Init {
				t.Fatalf("detection = %+v, want %s/%s/%s", res.Detection, f.Distro, f.Pkg, f.Init)
			}
			if res.GoArch != "linux-"+strings.TrimPrefix(res.Plan.Toolchain.Arch, "linux-") {
				t.Fatalf("resolved arch = %q, want %q", res.GoArch, res.Plan.Toolchain.Arch)
			}
			if res.ResolvedBranch == "" || len(res.ResolvedCommit) != 40 {
				t.Fatalf("resolved branch/commit incomplete: %s @ %s", res.ResolvedBranch, res.ResolvedCommit)
			}
			if res.StageBinary == "" || res.RecordFile == "" || res.ToolchainDir == "" {
				t.Fatalf("result paths incomplete: %+v", res)
			}

			client, err = h.Dial()
			if err != nil {
				t.Fatalf("redial: %v", err)
			}
			defer client.Close() //nolint:errcheck
			assertSourceInstallState(t, client, f.Name, res)

			// Idempotent rerun: succeeds, resolves the same branch,
			// reuses the toolchain, and leaks no temp files.
			res2, err := installer.SourceInstall(opts)
			if err != nil {
				t.Fatalf("source install rerun on %s: %v", f.Name, err)
			}
			if res2.ResolvedBranch != res.ResolvedBranch || len(res2.ResolvedCommit) != 40 {
				t.Fatalf("rerun resolved = %s @ %s, want branch %s", res2.ResolvedBranch, res2.ResolvedCommit, res.ResolvedBranch)
			}
			assertSourceInstallState(t, client, f.Name, res2)
			if out, err := RunCommand(client, "leak=0; ls /tmp | grep -q rsmon-toolchain && leak=1; ls /usr/local 2>/dev/null | grep -qE 'go-staging|go-backup' && leak=1; [ \"$leak\" -eq 1 ] && echo LEAK || echo CLEAN"); err != nil || strings.TrimSpace(string(out)) != "CLEAN" {
				t.Fatalf("toolchain temp dirs leaked after rerun: %q, %v", out, err)
			}
		})
	}
}

// assertSourceInstallState verifies the remote side-effects of a source
// install: the toolchain reports the pinned version, the staging binary
// exists and reports the resolved commit, and the record file carries
// the resolved branch and commit.
func assertSourceInstallState(t *testing.T, client *ssh.Client, name string, res *installer.SourceInstallResult) {
	t.Helper()

	out, err := RunCommand(client, res.ToolchainDir+"/bin/go version")
	if err != nil {
		t.Fatalf("%s: toolchain missing: %v", name, err)
	}
	if !strings.Contains(string(out), "go"+res.Plan.Toolchain.Version) {
		t.Fatalf("%s: toolchain version = %q, want go%s", name, out, res.Plan.Toolchain.Version)
	}

	out, err = RunCommand(client, "test -x "+shellQuote(res.StageBinary)+" && echo BUILT")
	if err != nil || !strings.Contains(string(out), "BUILT") {
		t.Fatalf("%s: staging binary not present at %s: %q, %v", name, res.StageBinary, out, err)
	}
	out, err = RunCommand(client, shellQuote(res.StageBinary)+" --version")
	if err != nil {
		t.Fatalf("%s: staging binary --version: %v", name, err)
	}
	if !strings.Contains(string(out), "commit="+res.ResolvedCommit[:12]) {
		t.Fatalf("%s: staging binary reports commit %q, want short %s", name, out, res.ResolvedCommit[:12])
	}

	record, err := RunCommand(client, "cat "+shellQuote(res.RecordFile))
	if err != nil {
		t.Fatalf("%s: read record file: %v", name, err)
	}
	if !strings.Contains(string(record), "branch="+res.ResolvedBranch) || !strings.Contains(string(record), "commit="+res.ResolvedCommit) {
		t.Fatalf("%s: record file = %q, want branch=%s commit=%s", name, record, res.ResolvedBranch, res.ResolvedCommit)
	}
}

// TestSourceInstallDirtyCheckoutPreservesStaging is the work-package-3
// failure-atomicity test: after a successful install, dirtying the
// tracked working tree makes the next checkout fail closed. The rerun
// must report the checkout error without ever reaching the build step,
// leaving the previous staging binary and commit record byte-for-byte
// unchanged, and without leaking toolchain staging/backup directories.
func TestSourceInstallDirtyCheckoutPreservesStaging(t *testing.T) {
	SkipUnlessEnabled(t)

	f := Fixtures()[0] // alpine is the smallest fixture
	h, err := New("dirty-"+f.Name, f)
	if err != nil {
		t.Fatal(err)
	}
	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute)
	defer cancel()
	if err := h.Start(ctx); err != nil {
		t.Fatalf("start %s fixture: %v", f.Name, err)
	}
	t.Cleanup(func() {
		if err := h.Stop(); err != nil {
			t.Errorf("stop %s fixture: %v", f.Name, err)
		}
	})

	opts := installer.SourceInstallOptions{
		SSHOptions: installer.SSHOptions{
			Host:           "127.0.0.1",
			Port:           h.Port(),
			User:           f.UserOrDefault(),
			IdentityFile:   testKeyPath(),
			KnownHostsFile: h.KnownHostsPath(),
		},
		Activation: installer.ActivationOptions{Activate: false},
	}
	if repo := strings.TrimSpace(os.Getenv("RSMON_TEST_SOURCE_REPO")); repo != "" {
		opts.Repo = repo
	}
	res, err := installer.SourceInstall(opts)
	if err != nil {
		t.Fatalf("initial source install: %v", err)
	}

	client, err := h.Dial()
	if err != nil {
		t.Fatalf("dial: %v", err)
	}
	defer client.Close() //nolint:errcheck

	// Dirty a tracked file so the rerun's checkout refuses to proceed.
	// Makefile differs between master and master~1 (unlike go.mod).
	if _, err := RunCommand(client, "git -C "+shellQuote(res.Plan.BuildDir)+" checkout -q master~1 -- Makefile"); err != nil {
		t.Fatalf("dirty the working tree: %v", err)
	}
	beforeBinary, err := RunCommand(client, "sha256sum "+shellQuote(res.StageBinary))
	if err != nil {
		t.Fatal(err)
	}
	beforeRecord, err := RunCommand(client, "cat "+shellQuote(res.RecordFile))
	if err != nil {
		t.Fatal(err)
	}

	if _, err := installer.SourceInstall(opts); err == nil || !strings.Contains(err.Error(), "check out branch") {
		t.Fatalf("rerun err = %v, want checkout failure on dirty tree", err)
	}

	afterBinary, err := RunCommand(client, "sha256sum "+shellQuote(res.StageBinary))
	if err != nil {
		t.Fatal(err)
	}
	afterRecord, err := RunCommand(client, "cat "+shellQuote(res.RecordFile))
	if err != nil {
		t.Fatal(err)
	}
	if !bytes.Equal(bytes.TrimSpace(beforeBinary), bytes.TrimSpace(afterBinary)) {
		t.Fatalf("staging binary changed after failed rerun:\nbefore: %s\nafter: %s", beforeBinary, afterBinary)
	}
	if !bytes.Equal(bytes.TrimSpace(beforeRecord), bytes.TrimSpace(afterRecord)) {
		t.Fatalf("commit record changed after failed rerun:\nbefore: %s\nafter: %s", beforeRecord, afterRecord)
	}
	if out, err := RunCommand(client, "leak=0; ls /tmp | grep -q rsmon-toolchain && leak=1; ls /usr/local 2>/dev/null | grep -qE 'go-staging|go-backup' && leak=1; [ \"$leak\" -eq 1 ] && echo LEAK || echo CLEAN"); err != nil || strings.TrimSpace(string(out)) != "CLEAN" {
		t.Fatalf("toolchain staging leaked after failed rerun: %q, %v", out, err)
	}
}

// makeProber builds an sshinstall.FileProber over a live SSH session.
func makeProber(client *ssh.Client) sshinstall.FileProber {
	return func(paths ...string) map[string]bool {
		// The trailing `; true` keeps the shell exit status 0: the
		// last `[ -e "$p" ]` in the loop would otherwise set exit 1
		// when the final path is absent (as on Arch), which is not an
		// error for a probe.
		expr := "for p in " + strings.Join(paths, " ") + "; do [ -e \"$p\" ] && printf '%s\\n' \"$p\"; done; true"
		out, err := RunCommand(client, expr)
		if err != nil {
			return nil
		}
		present := make(map[string]bool, len(paths))
		for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") {
			if line = strings.TrimSpace(line); line != "" {
				present[line] = true
			}
		}
		return present
	}
}

func shellQuote(value string) string {
	return "'" + strings.ReplaceAll(value, "'", "'\\''") + "'"
}