test(installer): add OpenSSH distro harness

internal/installer/harness/harness.go

@@ -0,0 +1,599 @@
+// Package harness starts real OpenSSH containers for the source-install
+// tests. It is the reusable Docker/OpenSSH test harness from work
+// package 1 of docs/source-installation.md.
+//
+// The harness never mocks SSH: it builds a distro fixture image, runs an
+// OpenSSH server in a container, waits for real network readiness, and
+// connects with the golang.org/x/crypto/ssh library and known_hosts
+// verification semantics the worker installer uses. Tests opt in with
+// RSMON_TEST_DOCKER=1 so ordinary unit runs never pull or start Docker.
+package harness
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"fmt"
+	"net"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+
+	"golang.org/x/crypto/ssh"
+	"golang.org/x/crypto/ssh/knownhosts"
+
+	"rocketgit.ru/rsmon/worker/internal/sshinstall"
+)
+
+// Fixture describes one distro OpenSSH fixture the harness can start.
+type Fixture struct {
+	Name   string            // "alpine", "ubuntu", "arch"
+	Distro sshinstall.Distro // expected detected distro
+	Pkg    sshinstall.PackageManager
+	Init   sshinstall.InitSystem
+	Image  string // explicit image override; empty uses the default
+	User   string // SSH user to authenticate as; default "root"
+}
+
+// Fixtures returns the supported source-install distro fixtures.
+func Fixtures() []Fixture {
+	return []Fixture{
+		{Name: "alpine", Distro: sshinstall.DistroAlpine, Pkg: sshinstall.PkgApk, Init: sshinstall.InitOpenRC},
+		{Name: "ubuntu", Distro: sshinstall.DistroUbuntu, Pkg: sshinstall.PkgApt, Init: sshinstall.InitSystemd},
+		{Name: "arch", Distro: sshinstall.DistroArch, Pkg: sshinstall.PkgPacman, Init: sshinstall.InitSystemd},
+	}
+}
+
+// defaultImages maps fixture name to its base image. reg.rsxx.ru mirror
+// refs are used where the mirror caches the distro; Docker Hub refs are
+// the fallback for distros the mirror does not carry and are overridable
+// per fixture via RSMON_TEST_IMAGE_<NAME>.
+var defaultImages = map[string]string{
+	"alpine": "reg.rsxx.ru/library/alpine:3",
+	"ubuntu": "ubuntu:24.04",
+	"arch":   "archlinux:latest",
+}
+
+// ImageRef resolves the base image reference for a fixture. Precedence:
+// RSMON_TEST_IMAGE_<NAME> env > Fixture.Image > mirror-aware default.
+func (f Fixture) ImageRef() string {
+	envKey := "RSMON_TEST_IMAGE_" + strings.ToUpper(strings.ReplaceAll(f.Name, "-", "_"))
+	if v := strings.TrimSpace(os.Getenv(envKey)); v != "" {
+		return v
+	}
+	if f.Image != "" {
+		return f.Image
+	}
+	return defaultImages[f.Name]
+}
+
+// UserOrDefault returns the SSH user, defaulting to root.
+func (f Fixture) UserOrDefault() string {
+	if f.User != "" {
+		return f.User
+	}
+	return "root"
+}
+
+// DockerfilePath is the fixture's Dockerfile under the package testdata.
+func (f Fixture) DockerfilePath() string {
+	return filepath.Join("testdata", "fixtures", f.Name, "Dockerfile")
+}
+
+// BuildContext is the Docker build context that carries the shared keys.
+func (f Fixture) BuildContext() string {
+	return filepath.Join("testdata")
+}
+
+// Harness manages the lifecycle of one OpenSSH test container: build,
+// run, wait for readiness, capture the host key, and tear down reliably.
+// Every resource it creates - fixture image tag, container, network, and
+// the temp known_hosts directory - is unique to this instance and is
+// removed by Stop.
+type Harness struct {
+	Name    string
+	Fixture Fixture
+
+	mu          sync.Mutex
+	started     bool
+	suffix      string
+	containerID string
+	container   string
+	network     string
+	imageTag    string
+	port        int
+	user        string
+	keyPath     string
+	knownHosts  string
+}
+
+// New creates a harness for the named fixture. The name must be a safe,
+// short identifier; a random suffix makes the container, network, and
+// fixture-image tag unique to this instance so teardown never touches
+// another harness's resources.
+func New(name string, f Fixture) (*Harness, error) {
+	if err := validateName(name); err != nil {
+		return nil, err
+	}
+	suffix := strconv.FormatInt(time.Now().UnixNano(), 36)
+	return &Harness{
+		Name:      name,
+		Fixture:   f,
+		suffix:    suffix,
+		container: "rsmon-worker-test-" + sanitize(name) + "-" + suffix,
+		network:   "rsmon-worker-test-" + sanitize(name) + "-" + suffix,
+		imageTag:  "rsmon-worker-test/" + sanitize(f.Name) + "-" + suffix + ":local",
+		user:      f.UserOrDefault(),
+	}, nil
+}
+
+// ContainerID returns the running container id (after Start).
+func (h *Harness) ContainerID() string {
+	h.mu.Lock()
+	defer h.mu.Unlock()
+	return h.containerID
+}
+
+// ContainerName returns the Docker container name.
+func (h *Harness) ContainerName() string { return h.container }
+
+// NetworkName returns the Docker network name.
+func (h *Harness) NetworkName() string { return h.network }
+
+// ImageTag returns the fixture image tag this harness instance builds
+// and removes on Stop. Tags are unique per instance, so removing one
+// never deletes a shared base image or another harness's image.
+func (h *Harness) ImageTag() string { return h.imageTag }
+
+// KnownHostsPath returns the temp known_hosts file created by Start, or
+// "" before Start. The surrounding directory is removed by Stop.
+func (h *Harness) KnownHostsPath() string {
+	h.mu.Lock()
+	defer h.mu.Unlock()
+	return h.knownHosts
+}
+
+// Addr returns the dialable host:port of the published SSH listener.
+// Before Start it is empty.
+func (h *Harness) Addr() string {
+	h.mu.Lock()
+	defer h.mu.Unlock()
+	if h.port == 0 {
+		return ""
+	}
+	return net.JoinHostPort("127.0.0.1", strconv.Itoa(h.port))
+}
+
+// Start builds the fixture image, starts the container, waits for real
+// SSH readiness, captures the server host key into a temp known_hosts
+// file, and records the published port. Every error path cleans up the
+// container, network, image tag, and temp dir created so far; on
+// success the caller owns teardown via t.Cleanup(h.Stop) or a defer.
+func (h *Harness) Start(ctx context.Context) error {
+	if !Enabled() {
+		return errors.New("harness integration is disabled; set RSMON_TEST_DOCKER=1 to run Docker/OpenSSH tests")
+	}
+	image := h.Fixture.ImageRef()
+	if image == "" {
+		return fmt.Errorf("fixture %q has no base image", h.Fixture.Name)
+	}
+
+	// Any error after the first resource is created must release what
+	// was already allocated. Stop is idempotent and tolerates resources
+	// that were never created.
+	success := false
+	defer func() {
+		if !success {
+			_ = h.Stop()
+		}
+	}()
+
+	if _, err := dockerCmd(ctx, "build", "-q", "-t", h.imageTag, "-f", h.Fixture.DockerfilePath(), h.Fixture.BuildContext()); err != nil {
+		return fmt.Errorf("build %s fixture image: %w", h.Fixture.Name, err)
+	}
+	if _, err := dockerCmd(ctx, "network", "create", h.network); err != nil {
+		return fmt.Errorf("create test network: %w", err)
+	}
+	// docker run -d prints the container id directly, so no lookup is
+	// needed; the container name is the stable handle for later docker
+	// calls and the id is captured for diagnostics and assertions.
+	out, err := dockerCmd(
+		ctx, "run", "-d",
+		"--name", h.container,
+		"--network", h.network,
+		"-p", "127.0.0.1::22",
+		h.imageTag,
+	)
+	if err != nil {
+		return fmt.Errorf("start %s fixture container: %w", h.Fixture.Name, err)
+	}
+	h.containerID = strings.TrimSpace(out)
+
+	port, err := h.publishedPort(ctx)
+	if err != nil {
+		return err
+	}
+	h.port = port
+
+	key, err := h.waitForSSH(ctx, h.Addr())
+	if err != nil {
+		return fmt.Errorf("wait for ssh readiness on %s fixture: %w", h.Fixture.Name, err)
+	}
+	// Register the temp dir before writing so a write failure still
+	// leaves it known to the cleanup defer.
+	dir, err := os.MkdirTemp("", "rsmon-worker-harness-")
+	if err != nil {
+		return err
+	}
+	h.knownHosts = filepath.Join(dir, "known_hosts")
+	hostsFile, err := writeKnownHosts(dir, h.Addr(), key)
+	if err != nil {
+		return err
+	}
+	h.keyPath = testKeyPath()
+	h.knownHosts = hostsFile
+
+	h.mu.Lock()
+	h.started = true
+	h.mu.Unlock()
+	success = true
+	return nil
+}
+
+// Stop releases every resource the harness created: the container, its
+// dedicated network, the per-instance fixture image tag (never a shared
+// base image), and the temp known_hosts directory. It is idempotent and
+// tolerates already-removed resources so teardown never fails the test
+// twice.
+func (h *Harness) Stop() error {
+	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
+	defer cancel()
+	var errs []error
+	if h.container != "" {
+		if _, err := dockerCmd(ctx, "rm", "-f", h.container); err != nil && !isNotExist(err) {
+			errs = append(errs, err)
+		}
+	}
+	if h.network != "" {
+		if _, err := dockerCmd(ctx, "network", "rm", h.network); err != nil && !isNotExist(err) {
+			errs = append(errs, err)
+		}
+	}
+	if h.imageTag != "" {
+		if _, err := dockerCmd(ctx, "image", "rm", h.imageTag); err != nil && !isNotExist(err) && !isInUse(err) {
+			errs = append(errs, err)
+		}
+	}
+	if h.knownHosts != "" {
+		if err := os.RemoveAll(filepath.Dir(h.knownHosts)); err != nil {
+			errs = append(errs, fmt.Errorf("remove harness temp dir: %w", err))
+		}
+	}
+	h.mu.Lock()
+	h.started = false
+	h.containerID = ""
+	h.port = 0
+	h.keyPath = ""
+	h.knownHosts = ""
+	h.mu.Unlock()
+	return errors.Join(errs...)
+}
+
+// Dial connects using the same SSH library (golang.org/x/crypto/ssh) and
+// the same known_hosts host-key verification the installer's deploy path
+// relies on. The harness owns this code rather than calling into the
+// installer package so tests stay independent; only the library and the
+// verification semantics are shared.
+func (h *Harness) Dial() (*ssh.Client, error) {
+	h.mu.Lock()
+	if !h.started || h.knownHosts == "" || h.keyPath == "" {
+		h.mu.Unlock()
+		return nil, errors.New("harness is not started")
+	}
+	addr := net.JoinHostPort("127.0.0.1", strconv.Itoa(h.port))
+	knownHostsFile, keyPath, user := h.knownHosts, h.keyPath, h.user
+	h.mu.Unlock()
+
+	keyBytes, err := os.ReadFile(keyPath)
+	if err != nil {
+		return nil, err
+	}
+	signer, err := ssh.ParsePrivateKey(keyBytes)
+	if err != nil {
+		return nil, err
+	}
+	hostKeyCallback, err := knownhosts.New(knownHostsFile)
+	if err != nil {
+		return nil, err
+	}
+	config := &ssh.ClientConfig{
+		User:            user,
+		Auth:            []ssh.AuthMethod{ssh.PublicKeys(signer)},
+		HostKeyCallback: hostKeyCallback,
+		Timeout:         15 * time.Second,
+	}
+	return ssh.Dial("tcp", addr, config)
+}
+
+// RunCommand executes a command over an existing SSH client and returns
+// its stdout. Stderr is folded into the returned error message on
+// failure so test failures show what went wrong.
+func RunCommand(client *ssh.Client, command string) ([]byte, error) {
+	session, err := client.NewSession()
+	if err != nil {
+		return nil, err
+	}
+	defer session.Close() //nolint:errcheck
+	var stdout, stderr bytes.Buffer
+	session.Stdout = &stdout
+	session.Stderr = &stderr
+	if err := session.Run(command); err != nil {
+		if msg := strings.TrimSpace(stderr.String()); msg != "" {
+			return stdout.Bytes(), fmt.Errorf("%w: %s", err, msg)
+		}
+		return stdout.Bytes(), err
+	}
+	return stdout.Bytes(), nil
+}
+
+// DockerCmd runs the Docker CLI with the given arguments, returning
+// stdout. Exposed so integration tests can assert teardown state.
+func (h *Harness) DockerCmd(ctx context.Context, args ...string) (string, error) {
+	return dockerCmd(ctx, args...)
+}
+
+// Enabled reports whether Docker-backed integration tests may run. The
+// opt-in env var keeps ordinary `go test` and CI unit runs from pulling
+// or starting any container.
+func Enabled() bool {
+	switch strings.ToLower(strings.TrimSpace(os.Getenv("RSMON_TEST_DOCKER"))) {
+	case "1", "true", "yes", "on":
+		return true
+	}
+	return false
+}
+
+// SkipUnlessEnabled skips the test with a helpful message unless the
+// Docker integration opt-in is set.
+func SkipUnlessEnabled(t interface {
+	Helper()
+	Skipf(format string, args ...interface{})
+},
+) {
+	t.Helper()
+	if !Enabled() {
+		t.Skipf("Docker/OpenSSH harness tests are opt-in; set RSMON_TEST_DOCKER=1 to run them")
+	}
+}
+
+// dockerBin is the Docker CLI binary. Overridable in tests via
+// SetDockerBin so unit tests can stub the harness's docker calls.
+var dockerBin = "docker"
+
+// SetDockerBin overrides the Docker CLI binary used by the harness.
+// Pass an empty value to restore the default ("docker").
+func SetDockerBin(name string) {
+	if name == "" {
+		dockerBin = "docker"
+		return
+	}
+	dockerBin = name
+}
+
+func dockerCmd(ctx context.Context, args ...string) (string, error) {
+	cmd := exec.CommandContext(ctx, dockerBin, args...)
+	out, err := cmd.Output()
+	if err != nil {
+		if ee, ok := err.(*exec.ExitError); ok && len(ee.Stderr) > 0 {
+			return "", fmt.Errorf("docker %s: %s", strings.Join(args, " "), strings.TrimSpace(string(ee.Stderr)))
+		}
+		return "", fmt.Errorf("docker %s: %w", strings.Join(args, " "), err)
+	}
+	return string(out), nil
+}
+
+// isNotExist reports whether the docker error is a missing resource
+// (container/network/image already removed or never created). Docker
+// reports missing networks and images as "... not found" and missing
+// containers as "No such container: ...".
+func isNotExist(err error) bool {
+	if err == nil {
+		return false
+	}
+	lower := strings.ToLower(err.Error())
+	for _, marker := range []string{"no such container", "no such network", "no such image", "is not running", "not found"} {
+		if strings.Contains(lower, marker) {
+			return true
+		}
+	}
+	return false
+}
+
+// isInUse reports whether a docker image removal failed because another
+// container or tag still references the image. Teardown must not treat
+// that as a leak error: the per-instance tags make this unlikely, but
+// tolerating it keeps Stop deterministic under concurrent harnesses.
+func isInUse(err error) bool {
+	if err == nil {
+		return false
+	}
+	lower := strings.ToLower(err.Error())
+	for _, marker := range []string{"image is being used", "image is in use", "image is referenced"} {
+		if strings.Contains(lower, marker) {
+			return true
+		}
+	}
+	return false
+}
+
+// publishedPort queries the host port docker published for the
+// container's port 22.
+func (h *Harness) publishedPort(ctx context.Context) (int, error) {
+	out, err := dockerCmd(ctx, "port", h.container, "22/tcp")
+	if err != nil {
+		return 0, err
+	}
+	return parsePublishedPort(out)
+}
+
+// parsePublishedPort extracts the host port from `docker port` output
+// like "127.0.0.1:49153", "0.0.0.0:49153", or "::1:49153".
+func parsePublishedPort(out string) (int, error) {
+	for _, line := range strings.Split(out, "\n") {
+		line = strings.TrimSpace(line)
+		if line == "" {
+			continue
+		}
+		sep := strings.LastIndex(line, ":")
+		if sep < 0 {
+			continue
+		}
+		port, err := strconv.Atoi(strings.TrimSpace(line[sep+1:]))
+		if err == nil && port > 0 && port <= 65535 {
+			return port, nil
+		}
+	}
+	return 0, fmt.Errorf("could not parse published port from %q", out)
+}
+
+// waitForSSH waits for the container's SSH listener to accept a real
+// handshake and returns the server host key. TCP readiness alone is not
+// enough; the ssh.Dial must succeed and the exec channel must answer.
+func (h *Harness) waitForSSH(ctx context.Context, addr string) (ssh.PublicKey, error) {
+	if err := waitForPort(ctx, addr, 60*time.Second); err != nil {
+		return nil, err
+	}
+	deadline := time.Now().Add(60 * time.Second)
+	var lastErr error
+	for {
+		if ctx.Err() != nil {
+			return nil, fmt.Errorf("ssh readiness: %w (last: %v)", ctx.Err(), lastErr)
+		}
+		if time.Now().After(deadline) {
+			return nil, fmt.Errorf("ssh readiness timed out (last: %v)", lastErr)
+		}
+		key, err := h.handshake(addr)
+		if err == nil {
+			return key, nil
+		}
+		lastErr = err
+		time.Sleep(500 * time.Millisecond)
+	}
+}
+
+// handshake performs one full SSH exchange and captures the server host
+// key via the callback so the harness can write a known_hosts entry.
+// A fresh known_hosts file trusts the first key it sees (trust-on-first-
+// use); the host-key-mismatch integration test proves that a known_hosts
+// entry carrying a *different* key is rejected before any command runs.
+func (h *Harness) handshake(addr string) (ssh.PublicKey, error) {
+	keyBytes, err := os.ReadFile(testKeyPath())
+	if err != nil {
+		return nil, err
+	}
+	signer, err := ssh.ParsePrivateKey(keyBytes)
+	if err != nil {
+		return nil, err
+	}
+	var got ssh.PublicKey
+	config := &ssh.ClientConfig{
+		User: h.user,
+		Auth: []ssh.AuthMethod{ssh.PublicKeys(signer)},
+		HostKeyCallback: func(_ string, _ net.Addr, key ssh.PublicKey) error {
+			got = key
+			return nil
+		},
+		Timeout: 5 * time.Second,
+	}
+	client, err := ssh.Dial("tcp", addr, config)
+	if err != nil {
+		return nil, err
+	}
+	defer client.Close() //nolint:errcheck
+	session, err := client.NewSession()
+	if err != nil {
+		return nil, err
+	}
+	defer session.Close() //nolint:errcheck
+	if _, err := session.Output("true"); err != nil {
+		return nil, err
+	}
+	if got == nil {
+		return nil, errors.New("no host key returned by handshake")
+	}
+	return got, nil
+}
+
+// waitForPort polls a TCP endpoint until it accepts a connection. It is
+// a plain network probe; SSH readiness is separately verified.
+func waitForPort(ctx context.Context, addr string, timeout time.Duration) error {
+	deadline := time.Now().Add(timeout)
+	for {
+		if ctx.Err() != nil {
+			return ctx.Err()
+		}
+		if time.Now().After(deadline) {
+			return fmt.Errorf("port %s not open within %s", addr, timeout)
+		}
+		conn, err := net.DialTimeout("tcp", addr, 2*time.Second)
+		if err == nil {
+			_ = conn.Close()
+			return nil
+		}
+		time.Sleep(500 * time.Millisecond)
+	}
+}
+
+// writeKnownHosts writes a known_hosts entry for the harness address
+// inside an existing temp directory, using the captured server host key.
+func writeKnownHosts(dir, addr string, key ssh.PublicKey) (string, error) {
+	path := filepath.Join(dir, "known_hosts")
+	line := knownhosts.Line([]string{addr}, key)
+	if err := os.WriteFile(path, []byte(line+"\n"), 0o600); err != nil {
+		return "", err
+	}
+	return path, nil
+}
+
+// testKeyPath returns the shared test private key bundled with the
+// package. The fixtures bake the matching public key into authorized_keys.
+//
+// This keypair is strictly test-only: it grants root SSH access only to
+// the disposable fixture containers that bake its public key. It must
+// never be used for real hosts, copied into production images, or
+// treated as a credential anywhere outside the harness.
+func testKeyPath() string {
+	return filepath.Join("testdata", "keys", "rsmon_test_ed25519")
+}
+
+// validateName rejects harness names that could inject shell or docker
+// metacharacters into container/network names.
+func validateName(name string) error {
+	if name == "" || len(name) > 64 {
+		return errors.New("harness name must be 1-64 characters")
+	}
+	for _, r := range name {
+		lower := r >= 'a' && r <= 'z'
+		digit := r >= '0' && r <= '9'
+		if !lower && !digit && r != '-' {
+			return fmt.Errorf("harness name %q must be lowercase alphanumeric and hyphens", name)
+		}
+	}
+	return nil
+}
+
+func sanitize(name string) string {
+	var b strings.Builder
+	for _, r := range name {
+		if r >= 'a' && r <= 'z' || r >= '0' && r <= '9' {
+			b.WriteRune(r)
+		} else {
+			b.WriteByte('-')
+		}
+	}
+	return b.String()
+}