worker/internal/installer/harness/harness.go

// Package harness starts real OpenSSH containers for the source-install
// tests. It is the reusable Docker/OpenSSH test harness from work
// package 1 of docs/source-installation.md.
//
// The harness never mocks SSH: it builds a distro fixture image, runs an
// OpenSSH server in a container, waits for real network readiness, and
// connects with the golang.org/x/crypto/ssh library and known_hosts
// verification semantics the worker installer uses. Tests opt in with
// RSMON_TEST_DOCKER=1 so ordinary unit runs never pull or start Docker.
package harness

import (
	"bytes"
	"context"
	"errors"
	"fmt"
	"net"
	"os"
	"os/exec"
	"path/filepath"
	"strconv"
	"strings"
	"sync"
	"time"

	"golang.org/x/crypto/ssh"
	"golang.org/x/crypto/ssh/knownhosts"

	"rocketgit.ru/rsmon/worker/internal/sshinstall"
)

// Fixture describes one distro OpenSSH fixture the harness can start.
type Fixture struct {
	Name   string            // "alpine", "ubuntu", "arch"
	Distro sshinstall.Distro // expected detected distro
	Pkg    sshinstall.PackageManager
	Init   sshinstall.InitSystem
	Image  string // explicit image override; empty uses the default
	User   string // SSH user to authenticate as; default "root"
}

// Fixtures returns the supported source-install distro fixtures.
func Fixtures() []Fixture {
	return []Fixture{
		{Name: "alpine", Distro: sshinstall.DistroAlpine, Pkg: sshinstall.PkgApk, Init: sshinstall.InitOpenRC},
		{Name: "ubuntu", Distro: sshinstall.DistroUbuntu, Pkg: sshinstall.PkgApt, Init: sshinstall.InitSystemd},
		{Name: "arch", Distro: sshinstall.DistroArch, Pkg: sshinstall.PkgPacman, Init: sshinstall.InitSystemd},
	}
}

// defaultImages maps fixture name to its base image. reg.rsxx.ru mirror
// refs are used where the mirror caches the distro; Docker Hub refs are
// the fallback for distros the mirror does not carry and are overridable
// per fixture via RSMON_TEST_IMAGE_<NAME>.
var defaultImages = map[string]string{
	"alpine": "reg.rsxx.ru/library/alpine:3",
	"ubuntu": "ubuntu:24.04",
	"arch":   "archlinux:latest",
}

// ImageRef resolves the base image reference for a fixture. Precedence:
// RSMON_TEST_IMAGE_<NAME> env > Fixture.Image > mirror-aware default.
func (f Fixture) ImageRef() string {
	envKey := "RSMON_TEST_IMAGE_" + strings.ToUpper(strings.ReplaceAll(f.Name, "-", "_"))
	if v := strings.TrimSpace(os.Getenv(envKey)); v != "" {
		return v
	}
	if f.Image != "" {
		return f.Image
	}
	return defaultImages[f.Name]
}

// UserOrDefault returns the SSH user, defaulting to root.
func (f Fixture) UserOrDefault() string {
	if f.User != "" {
		return f.User
	}
	return "root"
}

// DockerfilePath is the fixture's Dockerfile under the package testdata.
func (f Fixture) DockerfilePath() string {
	return filepath.Join("testdata", "fixtures", f.Name, "Dockerfile")
}

// BuildContext is the Docker build context that carries the shared keys.
func (f Fixture) BuildContext() string {
	return filepath.Join("testdata")
}

// Harness manages the lifecycle of one OpenSSH test container: build,
// run, wait for readiness, capture the host key, and tear down reliably.
// Every resource it creates - fixture image tag, container, network, and
// the temp known_hosts directory - is unique to this instance and is
// removed by Stop.
type Harness struct {
	Name    string
	Fixture Fixture

	mu          sync.Mutex
	started     bool
	suffix      string
	containerID string
	container   string
	network     string
	imageTag    string
	port        int
	user        string
	keyPath     string
	knownHosts  string
}

// New creates a harness for the named fixture. The name must be a safe,
// short identifier; a random suffix makes the container, network, and
// fixture-image tag unique to this instance so teardown never touches
// another harness's resources.
func New(name string, f Fixture) (*Harness, error) {
	if err := validateName(name); err != nil {
		return nil, err
	}
	suffix := strconv.FormatInt(time.Now().UnixNano(), 36)
	return &Harness{
		Name:      name,
		Fixture:   f,
		suffix:    suffix,
		container: "rsmon-worker-test-" + sanitize(name) + "-" + suffix,
		network:   "rsmon-worker-test-" + sanitize(name) + "-" + suffix,
		imageTag:  "rsmon-worker-test/" + sanitize(f.Name) + "-" + suffix + ":local",
		user:      f.UserOrDefault(),
	}, nil
}

// ContainerID returns the running container id (after Start).
func (h *Harness) ContainerID() string {
	h.mu.Lock()
	defer h.mu.Unlock()
	return h.containerID
}

// ContainerName returns the Docker container name.
func (h *Harness) ContainerName() string { return h.container }

// NetworkName returns the Docker network name.
func (h *Harness) NetworkName() string { return h.network }

// ImageTag returns the fixture image tag this harness instance builds
// and removes on Stop. Tags are unique per instance, so removing one
// never deletes a shared base image or another harness's image.
func (h *Harness) ImageTag() string { return h.imageTag }

// KnownHostsPath returns the temp known_hosts file created by Start, or
// "" before Start. The surrounding directory is removed by Stop.
func (h *Harness) KnownHostsPath() string {
	h.mu.Lock()
	defer h.mu.Unlock()
	return h.knownHosts
}

// Addr returns the dialable host:port of the published SSH listener.
// Before Start it is empty.
func (h *Harness) Addr() string {
	h.mu.Lock()
	defer h.mu.Unlock()
	if h.port == 0 {
		return ""
	}
	return net.JoinHostPort("127.0.0.1", strconv.Itoa(h.port))
}

// Port returns the published host port of the container's SSH listener
// (the host is always 127.0.0.1). 0 before Start.
func (h *Harness) Port() int {
	h.mu.Lock()
	defer h.mu.Unlock()
	return h.port
}

// Start builds the fixture image, starts the container, waits for real
// SSH readiness, captures the server host key into a temp known_hosts
// file, and records the published port. Every error path cleans up the
// container, network, image tag, and temp dir created so far; on
// success the caller owns teardown via t.Cleanup(h.Stop) or a defer.
func (h *Harness) Start(ctx context.Context) error {
	if !Enabled() {
		return errors.New("harness integration is disabled; set RSMON_TEST_DOCKER=1 to run Docker/OpenSSH tests")
	}
	image := h.Fixture.ImageRef()
	if image == "" {
		return fmt.Errorf("fixture %q has no base image", h.Fixture.Name)
	}

	// Any error after the first resource is created must release what
	// was already allocated. Stop is idempotent and tolerates resources
	// that were never created.
	success := false
	defer func() {
		if !success {
			_ = h.Stop()
		}
	}()

	if _, err := dockerCmd(ctx, "build", "-q", "-t", h.imageTag, "-f", h.Fixture.DockerfilePath(), h.Fixture.BuildContext()); err != nil {
		return fmt.Errorf("build %s fixture image: %w", h.Fixture.Name, err)
	}
	if _, err := dockerCmd(ctx, "network", "create", h.network); err != nil {
		return fmt.Errorf("create test network: %w", err)
	}
	// docker run -d prints the container id directly, so no lookup is
	// needed; the container name is the stable handle for later docker
	// calls and the id is captured for diagnostics and assertions.
	runArgs := []string{"run", "-d", "--name", h.container, "--network", h.network, "-p", "127.0.0.1::22"}
	runArgs = append(runArgs, dockerDNS()...)
	runArgs = append(runArgs, h.imageTag)
	out, err := dockerCmd(ctx, runArgs...)
	if err != nil {
		return fmt.Errorf("start %s fixture container: %w", h.Fixture.Name, err)
	}
	h.containerID = strings.TrimSpace(out)

	port, err := h.publishedPort(ctx)
	if err != nil {
		return err
	}
	h.port = port

	key, err := h.waitForSSH(ctx, h.Addr())
	if err != nil {
		return fmt.Errorf("wait for ssh readiness on %s fixture: %w", h.Fixture.Name, err)
	}
	// Register the temp dir before writing so a write failure still
	// leaves it known to the cleanup defer.
	dir, err := os.MkdirTemp("", "rsmon-worker-harness-")
	if err != nil {
		return err
	}
	h.knownHosts = filepath.Join(dir, "known_hosts")
	hostsFile, err := writeKnownHosts(dir, h.Addr(), key)
	if err != nil {
		return err
	}
	h.keyPath = testKeyPath()
	h.knownHosts = hostsFile

	h.mu.Lock()
	h.started = true
	h.mu.Unlock()
	success = true
	return nil
}

// dockerDNS returns the `--dns` arguments to pin for fixture containers,
// parsed from the comma-separated RSMON_TEST_DOCKER_DNS environment
// variable. It is empty by default (Docker's embedded DNS). The override
// exists so environments with flaky local resolvers can pin a reliable
// upstream for the internet-facing installs (go.dev, rocketgit.ru,
// proxy.golang.org), which would otherwise fail intermittently on DNS
// timeouts.
func dockerDNS() []string {
	var args []string
	for _, ns := range strings.Split(os.Getenv("RSMON_TEST_DOCKER_DNS"), ",") {
		if ns = strings.TrimSpace(ns); ns != "" {
			args = append(args, "--dns", ns)
		}
	}
	return args
}

// Stop releases every resource the harness created: the container, its
// dedicated network, the per-instance fixture image tag (never a shared
// base image), and the temp known_hosts directory. It is idempotent and
// tolerates already-removed resources so teardown never fails the test
// twice.
func (h *Harness) Stop() error {
	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
	defer cancel()
	var errs []error
	if h.container != "" {
		if _, err := dockerCmd(ctx, "rm", "-f", h.container); err != nil && !isNotExist(err) {
			errs = append(errs, err)
		}
	}
	if h.network != "" {
		if _, err := dockerCmd(ctx, "network", "rm", h.network); err != nil && !isNotExist(err) {
			errs = append(errs, err)
		}
	}
	if h.imageTag != "" {
		if _, err := dockerCmd(ctx, "image", "rm", h.imageTag); err != nil && !isNotExist(err) && !isInUse(err) {
			errs = append(errs, err)
		}
	}
	if h.knownHosts != "" {
		if err := os.RemoveAll(filepath.Dir(h.knownHosts)); err != nil {
			errs = append(errs, fmt.Errorf("remove harness temp dir: %w", err))
		}
	}
	h.mu.Lock()
	h.started = false
	h.containerID = ""
	h.port = 0
	h.keyPath = ""
	h.knownHosts = ""
	h.mu.Unlock()
	return errors.Join(errs...)
}

// Dial connects using the same SSH library (golang.org/x/crypto/ssh) and
// the same known_hosts host-key verification the installer's deploy path
// relies on. The harness owns this code rather than calling into the
// installer package so tests stay independent; only the library and the
// verification semantics are shared.
func (h *Harness) Dial() (*ssh.Client, error) {
	h.mu.Lock()
	if !h.started || h.knownHosts == "" || h.keyPath == "" {
		h.mu.Unlock()
		return nil, errors.New("harness is not started")
	}
	addr := net.JoinHostPort("127.0.0.1", strconv.Itoa(h.port))
	knownHostsFile, keyPath, user := h.knownHosts, h.keyPath, h.user
	h.mu.Unlock()

	keyBytes, err := os.ReadFile(keyPath)
	if err != nil {
		return nil, err
	}
	signer, err := ssh.ParsePrivateKey(keyBytes)
	if err != nil {
		return nil, err
	}
	hostKeyCallback, err := knownhosts.New(knownHostsFile)
	if err != nil {
		return nil, err
	}
	config := &ssh.ClientConfig{
		User:            user,
		Auth:            []ssh.AuthMethod{ssh.PublicKeys(signer)},
		HostKeyCallback: hostKeyCallback,
		Timeout:         15 * time.Second,
	}
	return ssh.Dial("tcp", addr, config)
}

// RunCommand executes a command over an existing SSH client and returns
// its stdout. Stderr is folded into the returned error message on
// failure so test failures show what went wrong.
func RunCommand(client *ssh.Client, command string) ([]byte, error) {
	session, err := client.NewSession()
	if err != nil {
		return nil, err
	}
	defer session.Close() //nolint:errcheck
	var stdout, stderr bytes.Buffer
	session.Stdout = &stdout
	session.Stderr = &stderr
	if err := session.Run(command); err != nil {
		if msg := strings.TrimSpace(stderr.String()); msg != "" {
			return stdout.Bytes(), fmt.Errorf("%w: %s", err, msg)
		}
		return stdout.Bytes(), err
	}
	return stdout.Bytes(), nil
}

// DockerCmd runs the Docker CLI with the given arguments, returning
// stdout. Exposed so integration tests can assert teardown state.
func (h *Harness) DockerCmd(ctx context.Context, args ...string) (string, error) {
	return dockerCmd(ctx, args...)
}

// Enabled reports whether Docker-backed integration tests may run. The
// opt-in env var keeps ordinary `go test` and CI unit runs from pulling
// or starting any container.
func Enabled() bool {
	switch strings.ToLower(strings.TrimSpace(os.Getenv("RSMON_TEST_DOCKER"))) {
	case "1", "true", "yes", "on":
		return true
	}
	return false
}

// SkipUnlessEnabled skips the test with a helpful message unless the
// Docker integration opt-in is set.
func SkipUnlessEnabled(t interface {
	Helper()
	Skipf(format string, args ...interface{})
},
) {
	t.Helper()
	if !Enabled() {
		t.Skipf("Docker/OpenSSH harness tests are opt-in; set RSMON_TEST_DOCKER=1 to run them")
	}
}

// dockerBin is the Docker CLI binary. Overridable in tests via
// SetDockerBin so unit tests can stub the harness's docker calls.
var dockerBin = "docker"

// SetDockerBin overrides the Docker CLI binary used by the harness.
// Pass an empty value to restore the default ("docker").
func SetDockerBin(name string) {
	if name == "" {
		dockerBin = "docker"
		return
	}
	dockerBin = name
}

func dockerCmd(ctx context.Context, args ...string) (string, error) {
	cmd := exec.CommandContext(ctx, dockerBin, args...)
	out, err := cmd.Output()
	if err != nil {
		if ee, ok := err.(*exec.ExitError); ok && len(ee.Stderr) > 0 {
			return "", fmt.Errorf("docker %s: %s", strings.Join(args, " "), strings.TrimSpace(string(ee.Stderr)))
		}
		return "", fmt.Errorf("docker %s: %w", strings.Join(args, " "), err)
	}
	return string(out), nil
}

// isNotExist reports whether the docker error is a missing resource
// (container/network/image already removed or never created). Docker
// reports missing networks and images as "... not found" and missing
// containers as "No such container: ...".
func isNotExist(err error) bool {
	if err == nil {
		return false
	}
	lower := strings.ToLower(err.Error())
	for _, marker := range []string{"no such container", "no such network", "no such image", "is not running", "not found"} {
		if strings.Contains(lower, marker) {
			return true
		}
	}
	return false
}

// isInUse reports whether a docker image removal failed because another
// container or tag still references the image. Teardown must not treat
// that as a leak error: the per-instance tags make this unlikely, but
// tolerating it keeps Stop deterministic under concurrent harnesses.
func isInUse(err error) bool {
	if err == nil {
		return false
	}
	lower := strings.ToLower(err.Error())
	for _, marker := range []string{"image is being used", "image is in use", "image is referenced"} {
		if strings.Contains(lower, marker) {
			return true
		}
	}
	return false
}

// publishedPort queries the host port docker published for the
// container's port 22.
func (h *Harness) publishedPort(ctx context.Context) (int, error) {
	out, err := dockerCmd(ctx, "port", h.container, "22/tcp")
	if err != nil {
		return 0, err
	}
	return parsePublishedPort(out)
}

// parsePublishedPort extracts the host port from `docker port` output
// like "127.0.0.1:49153", "0.0.0.0:49153", or "::1:49153".
func parsePublishedPort(out string) (int, error) {
	for _, line := range strings.Split(out, "\n") {
		line = strings.TrimSpace(line)
		if line == "" {
			continue
		}
		sep := strings.LastIndex(line, ":")
		if sep < 0 {
			continue
		}
		port, err := strconv.Atoi(strings.TrimSpace(line[sep+1:]))
		if err == nil && port > 0 && port <= 65535 {
			return port, nil
		}
	}
	return 0, fmt.Errorf("could not parse published port from %q", out)
}

// waitForSSH waits for the container's SSH listener to accept a real
// handshake and returns the server host key. TCP readiness alone is not
// enough; the ssh.Dial must succeed and the exec channel must answer.
func (h *Harness) waitForSSH(ctx context.Context, addr string) (ssh.PublicKey, error) {
	if err := waitForPort(ctx, addr, 60*time.Second); err != nil {
		return nil, err
	}
	deadline := time.Now().Add(60 * time.Second)
	var lastErr error
	for {
		if ctx.Err() != nil {
			return nil, fmt.Errorf("ssh readiness: %w (last: %v)", ctx.Err(), lastErr)
		}
		if time.Now().After(deadline) {
			return nil, fmt.Errorf("ssh readiness timed out (last: %v)", lastErr)
		}
		key, err := h.handshake(addr)
		if err == nil {
			return key, nil
		}
		lastErr = err
		time.Sleep(500 * time.Millisecond)
	}
}

// handshake performs one full SSH exchange and captures the server host
// key via the callback so the harness can write a known_hosts entry.
// A fresh known_hosts file trusts the first key it sees (trust-on-first-
// use); the host-key-mismatch integration test proves that a known_hosts
// entry carrying a *different* key is rejected before any command runs.
func (h *Harness) handshake(addr string) (ssh.PublicKey, error) {
	keyBytes, err := os.ReadFile(testKeyPath())
	if err != nil {
		return nil, err
	}
	signer, err := ssh.ParsePrivateKey(keyBytes)
	if err != nil {
		return nil, err
	}
	var got ssh.PublicKey
	config := &ssh.ClientConfig{
		User: h.user,
		Auth: []ssh.AuthMethod{ssh.PublicKeys(signer)},
		HostKeyCallback: func(_ string, _ net.Addr, key ssh.PublicKey) error {
			got = key
			return nil
		},
		Timeout: 5 * time.Second,
	}
	client, err := ssh.Dial("tcp", addr, config)
	if err != nil {
		return nil, err
	}
	defer client.Close() //nolint:errcheck
	session, err := client.NewSession()
	if err != nil {
		return nil, err
	}
	defer session.Close() //nolint:errcheck
	if _, err := session.Output("true"); err != nil {
		return nil, err
	}
	if got == nil {
		return nil, errors.New("no host key returned by handshake")
	}
	return got, nil
}

// waitForPort polls a TCP endpoint until it accepts a connection. It is
// a plain network probe; SSH readiness is separately verified.
func waitForPort(ctx context.Context, addr string, timeout time.Duration) error {
	deadline := time.Now().Add(timeout)
	for {
		if ctx.Err() != nil {
			return ctx.Err()
		}
		if time.Now().After(deadline) {
			return fmt.Errorf("port %s not open within %s", addr, timeout)
		}
		conn, err := net.DialTimeout("tcp", addr, 2*time.Second)
		if err == nil {
			_ = conn.Close()
			return nil
		}
		time.Sleep(500 * time.Millisecond)
	}
}

// writeKnownHosts writes a known_hosts entry for the harness address
// inside an existing temp directory, using the captured server host key.
func writeKnownHosts(dir, addr string, key ssh.PublicKey) (string, error) {
	path := filepath.Join(dir, "known_hosts")
	line := knownhosts.Line([]string{addr}, key)
	if err := os.WriteFile(path, []byte(line+"\n"), 0o600); err != nil {
		return "", err
	}
	return path, nil
}

// testKeyPath returns the shared test private key bundled with the
// package. The fixtures bake the matching public key into authorized_keys.
//
// This keypair is strictly test-only: it grants root SSH access only to
// the disposable fixture containers that bake its public key. It must
// never be used for real hosts, copied into production images, or
// treated as a credential anywhere outside the harness.
func testKeyPath() string {
	return filepath.Join("testdata", "keys", "rsmon_test_ed25519")
}

// validateName rejects harness names that could inject shell or docker
// metacharacters into container/network names.
func validateName(name string) error {
	if name == "" || len(name) > 64 {
		return errors.New("harness name must be 1-64 characters")
	}
	for _, r := range name {
		lower := r >= 'a' && r <= 'z'
		digit := r >= '0' && r <= '9'
		if !lower && !digit && r != '-' {
			return fmt.Errorf("harness name %q must be lowercase alphanumeric and hyphens", name)
		}
	}
	return nil
}

func sanitize(name string) string {
	var b strings.Builder
	for _, r := range name {
		if r >= 'a' && r <= 'z' || r >= '0' && r <= '9' {
			b.WriteRune(r)
		} else {
			b.WriteByte('-')
		}
	}
	return b.String()
}