package harness import ( "bytes" "context" "crypto/ed25519" "crypto/rand" "fmt" "os" "path/filepath" "strings" "testing" "time" "golang.org/x/crypto/ssh" "golang.org/x/crypto/ssh/knownhosts" "rocketgit.ru/rsmon/worker/internal/installer" "rocketgit.ru/rsmon/worker/internal/sshinstall" ) // TestHarnessFixtures is the work-package-1 acceptance test: each distro // fixture starts a real OpenSSH container, the harness waits for real // network readiness, and the installer's Go SSH client connects, runs // commands, and tears the environment down. // // Opt-in: set RSMON_TEST_DOCKER=1 (see make test-ssh). func TestHarnessFixtures(t *testing.T) { SkipUnlessEnabled(t) for _, f := range Fixtures() { f := f t.Run(f.Name, func(t *testing.T) { h, err := New("fixture-"+f.Name, f) if err != nil { t.Fatal(err) } ctx, cancel := context.WithTimeout(context.Background(), 20*time.Minute) defer cancel() if err := h.Start(ctx); err != nil { t.Fatalf("start %s fixture: %v", f.Name, err) } t.Cleanup(func() { if err := h.Stop(); err != nil { t.Errorf("stop %s fixture: %v", f.Name, err) } }) client, err := h.Dial() if err != nil { t.Fatalf("dial %s fixture: %v", f.Name, err) } defer client.Close() //nolint:errcheck // 1. SSH command execution is real: round-trip a nonce. nonce := fmt.Sprintf("RSMON_SSH_OK_%d", time.Now().UnixNano()) out, err := RunCommand(client, "printf '%s' "+shellQuote(nonce)) if err != nil { t.Fatalf("ssh round trip: %v", err) } if strings.TrimSpace(string(out)) != nonce { t.Fatalf("ssh round trip = %q, want %q", out, nonce) } // 2. The clean target has no Go toolchain and no worker source. out, err = RunCommand(client, "command -v go || true; test ! -e /usr/local/go && echo NO_GO; test ! -e /opt/rsmon-worker-src && echo NO_SOURCE") if err != nil { t.Fatalf("clean-state probe: %v", err) } clean := string(out) if strings.Contains(clean, "/go") && !strings.Contains(clean, "NO_GO") { t.Fatalf("fixture unexpectedly has Go installed: %q", clean) } if !strings.Contains(clean, "NO_SOURCE") { t.Fatalf("fixture unexpectedly has worker source: %q", clean) } // 3. Distro detection over the real session matches the fixture. out, err = RunCommand(client, "cat /etc/os-release") if err != nil { t.Fatalf("read os-release: %v", err) } d := sshinstall.Detect(string(out), makeProber(client)) if d.Distro != f.Distro { t.Fatalf("detected distro = %q, want %q (%s)", d.Distro, f.Distro, d.Summarize()) } if d.PackageManager != f.Pkg { t.Fatalf("detected package manager = %q, want %q", d.PackageManager, f.Pkg) } if d.InitSystem != f.Init { t.Fatalf("detected init = %q, want %q", d.InitSystem, f.Init) } t.Logf("%s: %s", f.Name, d.Summarize()) // 4. A full source plan resolves for the detected host, // including the pinned Go toolchain for its real arch. out, err = RunCommand(client, "uname -m") if err != nil { t.Fatalf("uname -m: %v", err) } goarch, err := sshinstall.GoArch(strings.TrimSpace(string(out))) if err != nil { t.Fatalf("GoArch(%q): %v", out, err) } plan, err := sshinstall.PlanSource(d, sshinstall.SourceOptions{UnameM: strings.TrimSpace(string(out))}) if err != nil { t.Fatalf("PlanSource: %v", err) } if plan.Toolchain.Arch != "linux-"+goarch { t.Fatalf("plan toolchain %q does not match detected arch %q", plan.Toolchain.Arch, goarch) } if len(plan.Packages) == 0 || plan.Repo == "" { t.Fatalf("incomplete plan: %+v", plan) } steps := plan.Steps() if len(steps) != 6 { t.Fatalf("plan steps = %d, want 6", len(steps)) } }) } } // TestHarnessHostKeyMismatch verifies the security gate: a dial against // a known_hosts entry carrying a different host key must fail before any // command can run. func TestHarnessHostKeyMismatch(t *testing.T) { SkipUnlessEnabled(t) f := Fixtures()[0] // alpine is the smallest fixture h, err := New("hostkey-"+f.Name, f) if err != nil { t.Fatal(err) } ctx, cancel := context.WithTimeout(context.Background(), 20*time.Minute) defer cancel() if err := h.Start(ctx); err != nil { t.Fatalf("start fixture: %v", err) } t.Cleanup(func() { if err := h.Stop(); err != nil { t.Errorf("stop fixture: %v", err) } }) // Build a known_hosts entry with a different (freshly generated) key. _, priv, err := ed25519.GenerateKey(rand.Reader) if err != nil { t.Fatal(err) } signer, err := ssh.NewSignerFromKey(priv) if err != nil { t.Fatal(err) } wrongFile := filepath.Join(t.TempDir(), "known_hosts") line := knownhosts.Line([]string{h.Addr()}, signer.PublicKey()) if err := os.WriteFile(wrongFile, []byte(line+"\n"), 0o600); err != nil { t.Fatal(err) } keyBytes, err := os.ReadFile(testKeyPath()) if err != nil { t.Fatal(err) } keySigner, err := ssh.ParsePrivateKey(keyBytes) if err != nil { t.Fatal(err) } callback, err := knownhosts.New(wrongFile) if err != nil { t.Fatal(err) } config := &ssh.ClientConfig{ User: f.UserOrDefault(), Auth: []ssh.AuthMethod{ssh.PublicKeys(keySigner)}, HostKeyCallback: callback, Timeout: 15 * time.Second, } client, err := ssh.Dial("tcp", h.Addr(), config) if err == nil { client.Close() //nolint:errcheck t.Fatal("dial with a mismatched host key succeeded") } if !strings.Contains(err.Error(), "knownhosts") && !strings.Contains(err.Error(), "key") { t.Fatalf("host-key mismatch error = %v, want a key/host verification failure", err) } } // TestHarnessTeardown verifies Stop reliably removes the container, the // dedicated network, the per-instance fixture image tag (never a shared // base image), and the temp known_hosts directory. func TestHarnessTeardown(t *testing.T) { SkipUnlessEnabled(t) f := Fixtures()[0] h, err := New("teardown-"+f.Name, f) if err != nil { t.Fatal(err) } ctx, cancel := context.WithTimeout(context.Background(), 20*time.Minute) defer cancel() if err := h.Start(ctx); err != nil { t.Fatalf("start fixture: %v", err) } if h.ContainerID() == "" { t.Fatal("container id empty after start") } if h.KnownHostsPath() == "" { t.Fatal("known_hosts not created after start") } knownHostsDir := filepath.Dir(h.KnownHostsPath()) if err := h.Stop(); err != nil { t.Fatalf("stop: %v", err) } if _, err := h.DockerCmd(ctx, "inspect", h.ContainerName()); err == nil { t.Fatal("container still present after Stop") } if _, err := h.DockerCmd(ctx, "network", "inspect", h.NetworkName()); err == nil { t.Fatal("network still present after Stop") } if _, err := h.DockerCmd(ctx, "image", "inspect", h.ImageTag()); err == nil { t.Fatalf("fixture image tag %q still present after Stop", h.ImageTag()) } if _, err := os.Stat(knownHostsDir); !os.IsNotExist(err) { t.Fatalf("known_hosts temp dir %q still present after Stop: %v", knownHostsDir, err) } if h.KnownHostsPath() != "" { t.Fatalf("KnownHostsPath = %q after Stop, want empty", h.KnownHostsPath()) } // Stop is idempotent. if err := h.Stop(); err != nil { t.Fatalf("second stop: %v", err) } } // TestHarnessHostKeyStableAcrossDial ensures the host key captured at // readiness is the one verified on every later dial, so a successful // Dial is proof of verified, real SSH transport. func TestHarnessHostKeyStable(t *testing.T) { SkipUnlessEnabled(t) f := Fixtures()[0] h, err := New("key-"+f.Name, f) if err != nil { t.Fatal(err) } ctx, cancel := context.WithTimeout(context.Background(), 20*time.Minute) defer cancel() if err := h.Start(ctx); err != nil { t.Fatalf("start fixture: %v", err) } t.Cleanup(func() { if err := h.Stop(); err != nil { t.Errorf("stop fixture: %v", err) } }) client, err := h.Dial() if err != nil { t.Fatalf("dial: %v", err) } client.Close() //nolint:errcheck client, err = h.Dial() if err != nil { t.Fatalf("second dial: %v", err) } defer client.Close() //nolint:errcheck if out, err := RunCommand(client, "echo VERIFIED"); err != nil || strings.TrimSpace(string(out)) != "VERIFIED" { t.Fatalf("verified session command = %q, %v", out, err) } } // TestSourceInstallFixtures is the work-package-3 acceptance test: each // distro fixture starts clean (no Go, no worker source) and the real // installer executes the full source flow over SSH - prerequisite // install, verified Go toolchain download/extraction, clone/update of the // public repository, resolved branch/commit record, and a build to a // staging path. The running service and its config are deliberately not // installed (that is work package 4). A rerun exercises idempotency. // // Opt-in: set RSMON_TEST_DOCKER=1 (see make test-ssh). func TestSourceInstallFixtures(t *testing.T) { SkipUnlessEnabled(t) for _, f := range Fixtures() { f := f t.Run(f.Name, func(t *testing.T) { h, err := New("source-"+f.Name, f) if err != nil { t.Fatal(err) } ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute) defer cancel() if err := h.Start(ctx); err != nil { t.Fatalf("start %s fixture: %v", f.Name, err) } t.Cleanup(func() { if err := h.Stop(); err != nil { t.Errorf("stop %s fixture: %v", f.Name, err) } }) // Clean state: no Go toolchain, no source, no leftover // toolchain temp dirs. client, err := h.Dial() if err != nil { t.Fatalf("dial %s fixture: %v", f.Name, err) } probe, err := RunCommand(client, "command -v go || true; test ! -e /usr/local/go && echo NO_GO; test ! -e /opt/rsmon-worker-src && echo NO_SOURCE; ls /tmp | grep -q rsmon-toolchain && echo LEAK; ls /usr/local 2>/dev/null | grep -qE 'go-staging|go-backup' && echo LEAK; echo DONE") if err != nil { t.Fatalf("clean-state probe: %v", err) } clean := string(probe) if !strings.Contains(clean, "NO_GO") || !strings.Contains(clean, "NO_SOURCE") { t.Fatalf("fixture is not clean: %q", clean) } if strings.Contains(clean, "LEAK") { t.Fatalf("fixture has leftover toolchain temp dirs: %q", clean) } client.Close() //nolint:errcheck opts := installer.SourceInstallOptions{ SSHOptions: installer.SSHOptions{ Host: "127.0.0.1", Port: h.Port(), User: f.UserOrDefault(), IdentityFile: testKeyPath(), KnownHostsFile: h.KnownHostsPath(), }, // Staging-only: service activation (work package 4) is // exercised by the dedicated activation tests. Activation: installer.ActivationOptions{Activate: false}, } if repo := strings.TrimSpace(os.Getenv("RSMON_TEST_SOURCE_REPO")); repo != "" { opts.Repo = repo } if branch := strings.TrimSpace(os.Getenv("RSMON_TEST_SOURCE_BRANCH")); branch != "" { opts.Branch = branch } res, err := installer.SourceInstall(opts) if err != nil { t.Fatalf("source install on %s: %v", f.Name, err) } t.Logf("%s: %s -> branch=%s commit=%s staged=%s", f.Name, res.Detection.Summarize(), res.ResolvedBranch, res.ResolvedCommit, res.StageBinary) if res.Detection.Distro != f.Distro || res.Detection.PackageManager != f.Pkg || res.Detection.InitSystem != f.Init { t.Fatalf("detection = %+v, want %s/%s/%s", res.Detection, f.Distro, f.Pkg, f.Init) } if res.GoArch != "linux-"+strings.TrimPrefix(res.Plan.Toolchain.Arch, "linux-") { t.Fatalf("resolved arch = %q, want %q", res.GoArch, res.Plan.Toolchain.Arch) } if res.ResolvedBranch == "" || len(res.ResolvedCommit) != 40 { t.Fatalf("resolved branch/commit incomplete: %s @ %s", res.ResolvedBranch, res.ResolvedCommit) } if res.StageBinary == "" || res.RecordFile == "" || res.ToolchainDir == "" { t.Fatalf("result paths incomplete: %+v", res) } client, err = h.Dial() if err != nil { t.Fatalf("redial: %v", err) } defer client.Close() //nolint:errcheck assertSourceInstallState(t, client, f.Name, res) // Idempotent rerun: succeeds, resolves the same branch, // reuses the toolchain, and leaks no temp files. res2, err := installer.SourceInstall(opts) if err != nil { t.Fatalf("source install rerun on %s: %v", f.Name, err) } if res2.ResolvedBranch != res.ResolvedBranch || len(res2.ResolvedCommit) != 40 { t.Fatalf("rerun resolved = %s @ %s, want branch %s", res2.ResolvedBranch, res2.ResolvedCommit, res.ResolvedBranch) } assertSourceInstallState(t, client, f.Name, res2) if out, err := RunCommand(client, "leak=0; ls /tmp | grep -q rsmon-toolchain && leak=1; ls /usr/local 2>/dev/null | grep -qE 'go-staging|go-backup' && leak=1; [ \"$leak\" -eq 1 ] && echo LEAK || echo CLEAN"); err != nil || strings.TrimSpace(string(out)) != "CLEAN" { t.Fatalf("toolchain temp dirs leaked after rerun: %q, %v", out, err) } }) } } // assertSourceInstallState verifies the remote side-effects of a source // install: the toolchain reports the pinned version, the staging binary // exists and reports the resolved commit, and the record file carries // the resolved branch and commit. func assertSourceInstallState(t *testing.T, client *ssh.Client, name string, res *installer.SourceInstallResult) { t.Helper() out, err := RunCommand(client, res.ToolchainDir+"/bin/go version") if err != nil { t.Fatalf("%s: toolchain missing: %v", name, err) } if !strings.Contains(string(out), "go"+res.Plan.Toolchain.Version) { t.Fatalf("%s: toolchain version = %q, want go%s", name, out, res.Plan.Toolchain.Version) } out, err = RunCommand(client, "test -x "+shellQuote(res.StageBinary)+" && echo BUILT") if err != nil || !strings.Contains(string(out), "BUILT") { t.Fatalf("%s: staging binary not present at %s: %q, %v", name, res.StageBinary, out, err) } out, err = RunCommand(client, shellQuote(res.StageBinary)+" --version") if err != nil { t.Fatalf("%s: staging binary --version: %v", name, err) } if !strings.Contains(string(out), "commit="+res.ResolvedCommit[:12]) { t.Fatalf("%s: staging binary reports commit %q, want short %s", name, out, res.ResolvedCommit[:12]) } record, err := RunCommand(client, "cat "+shellQuote(res.RecordFile)) if err != nil { t.Fatalf("%s: read record file: %v", name, err) } if !strings.Contains(string(record), "branch="+res.ResolvedBranch) || !strings.Contains(string(record), "commit="+res.ResolvedCommit) { t.Fatalf("%s: record file = %q, want branch=%s commit=%s", name, record, res.ResolvedBranch, res.ResolvedCommit) } } // TestSourceInstallDirtyCheckoutPreservesStaging is the work-package-3 // failure-atomicity test: after a successful install, dirtying the // tracked working tree makes the next checkout fail closed. The rerun // must report the checkout error without ever reaching the build step, // leaving the previous staging binary and commit record byte-for-byte // unchanged, and without leaking toolchain staging/backup directories. func TestSourceInstallDirtyCheckoutPreservesStaging(t *testing.T) { SkipUnlessEnabled(t) f := Fixtures()[0] // alpine is the smallest fixture h, err := New("dirty-"+f.Name, f) if err != nil { t.Fatal(err) } ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute) defer cancel() if err := h.Start(ctx); err != nil { t.Fatalf("start %s fixture: %v", f.Name, err) } t.Cleanup(func() { if err := h.Stop(); err != nil { t.Errorf("stop %s fixture: %v", f.Name, err) } }) opts := installer.SourceInstallOptions{ SSHOptions: installer.SSHOptions{ Host: "127.0.0.1", Port: h.Port(), User: f.UserOrDefault(), IdentityFile: testKeyPath(), KnownHostsFile: h.KnownHostsPath(), }, Activation: installer.ActivationOptions{Activate: false}, } if repo := strings.TrimSpace(os.Getenv("RSMON_TEST_SOURCE_REPO")); repo != "" { opts.Repo = repo } res, err := installer.SourceInstall(opts) if err != nil { t.Fatalf("initial source install: %v", err) } client, err := h.Dial() if err != nil { t.Fatalf("dial: %v", err) } defer client.Close() //nolint:errcheck // Dirty a tracked file so the rerun's checkout refuses to proceed. // Makefile differs between master and master~1 (unlike go.mod). if _, err := RunCommand(client, "git -C "+shellQuote(res.Plan.BuildDir)+" checkout -q master~1 -- Makefile"); err != nil { t.Fatalf("dirty the working tree: %v", err) } beforeBinary, err := RunCommand(client, "sha256sum "+shellQuote(res.StageBinary)) if err != nil { t.Fatal(err) } beforeRecord, err := RunCommand(client, "cat "+shellQuote(res.RecordFile)) if err != nil { t.Fatal(err) } if _, err := installer.SourceInstall(opts); err == nil || !strings.Contains(err.Error(), "check out branch") { t.Fatalf("rerun err = %v, want checkout failure on dirty tree", err) } afterBinary, err := RunCommand(client, "sha256sum "+shellQuote(res.StageBinary)) if err != nil { t.Fatal(err) } afterRecord, err := RunCommand(client, "cat "+shellQuote(res.RecordFile)) if err != nil { t.Fatal(err) } if !bytes.Equal(bytes.TrimSpace(beforeBinary), bytes.TrimSpace(afterBinary)) { t.Fatalf("staging binary changed after failed rerun:\nbefore: %s\nafter: %s", beforeBinary, afterBinary) } if !bytes.Equal(bytes.TrimSpace(beforeRecord), bytes.TrimSpace(afterRecord)) { t.Fatalf("commit record changed after failed rerun:\nbefore: %s\nafter: %s", beforeRecord, afterRecord) } if out, err := RunCommand(client, "leak=0; ls /tmp | grep -q rsmon-toolchain && leak=1; ls /usr/local 2>/dev/null | grep -qE 'go-staging|go-backup' && leak=1; [ \"$leak\" -eq 1 ] && echo LEAK || echo CLEAN"); err != nil || strings.TrimSpace(string(out)) != "CLEAN" { t.Fatalf("toolchain staging leaked after failed rerun: %q, %v", out, err) } } // makeProber builds an sshinstall.FileProber over a live SSH session. func makeProber(client *ssh.Client) sshinstall.FileProber { return func(paths ...string) map[string]bool { // The trailing `; true` keeps the shell exit status 0: the // last `[ -e "$p" ]` in the loop would otherwise set exit 1 // when the final path is absent (as on Arch), which is not an // error for a probe. expr := "for p in " + strings.Join(paths, " ") + "; do [ -e \"$p\" ] && printf '%s\\n' \"$p\"; done; true" out, err := RunCommand(client, expr) if err != nil { return nil } present := make(map[string]bool, len(paths)) for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") { if line = strings.TrimSpace(line); line != "" { present[line] = true } } return present } } func shellQuote(value string) string { return "'" + strings.ReplaceAll(value, "'", "'\\''") + "'" }