// Package harness starts real OpenSSH containers for the source-install // tests. It is the reusable Docker/OpenSSH test harness from work // package 1 of docs/source-installation.md. // // The harness never mocks SSH: it builds a distro fixture image, runs an // OpenSSH server in a container, waits for real network readiness, and // connects with the golang.org/x/crypto/ssh library and known_hosts // verification semantics the worker installer uses. Tests opt in with // RSMON_TEST_DOCKER=1 so ordinary unit runs never pull or start Docker. package harness import ( "bytes" "context" "errors" "fmt" "net" "os" "os/exec" "path/filepath" "strconv" "strings" "sync" "time" "golang.org/x/crypto/ssh" "golang.org/x/crypto/ssh/knownhosts" "rocketgit.ru/rsmon/worker/internal/sshinstall" ) // Fixture describes one distro OpenSSH fixture the harness can start. type Fixture struct { Name string // "alpine", "ubuntu", "arch" Distro sshinstall.Distro // expected detected distro Pkg sshinstall.PackageManager Init sshinstall.InitSystem Image string // explicit image override; empty uses the default User string // SSH user to authenticate as; default "root" } // Fixtures returns the supported source-install distro fixtures. func Fixtures() []Fixture { return []Fixture{ {Name: "alpine", Distro: sshinstall.DistroAlpine, Pkg: sshinstall.PkgApk, Init: sshinstall.InitOpenRC}, {Name: "ubuntu", Distro: sshinstall.DistroUbuntu, Pkg: sshinstall.PkgApt, Init: sshinstall.InitSystemd}, {Name: "arch", Distro: sshinstall.DistroArch, Pkg: sshinstall.PkgPacman, Init: sshinstall.InitSystemd}, } } // defaultImages maps fixture name to its base image. reg.rsxx.ru mirror // refs are used where the mirror caches the distro; Docker Hub refs are // the fallback for distros the mirror does not carry and are overridable // per fixture via RSMON_TEST_IMAGE_. var defaultImages = map[string]string{ "alpine": "reg.rsxx.ru/library/alpine:3", "ubuntu": "ubuntu:24.04", "arch": "archlinux:latest", } // ImageRef resolves the base image reference for a fixture. Precedence: // RSMON_TEST_IMAGE_ env > Fixture.Image > mirror-aware default. func (f Fixture) ImageRef() string { envKey := "RSMON_TEST_IMAGE_" + strings.ToUpper(strings.ReplaceAll(f.Name, "-", "_")) if v := strings.TrimSpace(os.Getenv(envKey)); v != "" { return v } if f.Image != "" { return f.Image } return defaultImages[f.Name] } // UserOrDefault returns the SSH user, defaulting to root. func (f Fixture) UserOrDefault() string { if f.User != "" { return f.User } return "root" } // DockerfilePath is the fixture's Dockerfile under the package testdata. func (f Fixture) DockerfilePath() string { return filepath.Join("testdata", "fixtures", f.Name, "Dockerfile") } // BuildContext is the Docker build context that carries the shared keys. func (f Fixture) BuildContext() string { return filepath.Join("testdata") } // Harness manages the lifecycle of one OpenSSH test container: build, // run, wait for readiness, capture the host key, and tear down reliably. // Every resource it creates - fixture image tag, container, network, and // the temp known_hosts directory - is unique to this instance and is // removed by Stop. type Harness struct { Name string Fixture Fixture mu sync.Mutex started bool suffix string containerID string container string network string imageTag string port int user string keyPath string knownHosts string } // New creates a harness for the named fixture. The name must be a safe, // short identifier; a random suffix makes the container, network, and // fixture-image tag unique to this instance so teardown never touches // another harness's resources. func New(name string, f Fixture) (*Harness, error) { if err := validateName(name); err != nil { return nil, err } suffix := strconv.FormatInt(time.Now().UnixNano(), 36) return &Harness{ Name: name, Fixture: f, suffix: suffix, container: "rsmon-worker-test-" + sanitize(name) + "-" + suffix, network: "rsmon-worker-test-" + sanitize(name) + "-" + suffix, imageTag: "rsmon-worker-test/" + sanitize(f.Name) + "-" + suffix + ":local", user: f.UserOrDefault(), }, nil } // ContainerID returns the running container id (after Start). func (h *Harness) ContainerID() string { h.mu.Lock() defer h.mu.Unlock() return h.containerID } // ContainerName returns the Docker container name. func (h *Harness) ContainerName() string { return h.container } // NetworkName returns the Docker network name. func (h *Harness) NetworkName() string { return h.network } // ImageTag returns the fixture image tag this harness instance builds // and removes on Stop. Tags are unique per instance, so removing one // never deletes a shared base image or another harness's image. func (h *Harness) ImageTag() string { return h.imageTag } // KnownHostsPath returns the temp known_hosts file created by Start, or // "" before Start. The surrounding directory is removed by Stop. func (h *Harness) KnownHostsPath() string { h.mu.Lock() defer h.mu.Unlock() return h.knownHosts } // Addr returns the dialable host:port of the published SSH listener. // Before Start it is empty. func (h *Harness) Addr() string { h.mu.Lock() defer h.mu.Unlock() if h.port == 0 { return "" } return net.JoinHostPort("127.0.0.1", strconv.Itoa(h.port)) } // Port returns the published host port of the container's SSH listener // (the host is always 127.0.0.1). 0 before Start. func (h *Harness) Port() int { h.mu.Lock() defer h.mu.Unlock() return h.port } // Start builds the fixture image, starts the container, waits for real // SSH readiness, captures the server host key into a temp known_hosts // file, and records the published port. Every error path cleans up the // container, network, image tag, and temp dir created so far; on // success the caller owns teardown via t.Cleanup(h.Stop) or a defer. func (h *Harness) Start(ctx context.Context) error { if !Enabled() { return errors.New("harness integration is disabled; set RSMON_TEST_DOCKER=1 to run Docker/OpenSSH tests") } image := h.Fixture.ImageRef() if image == "" { return fmt.Errorf("fixture %q has no base image", h.Fixture.Name) } // Any error after the first resource is created must release what // was already allocated. Stop is idempotent and tolerates resources // that were never created. success := false defer func() { if !success { _ = h.Stop() } }() if _, err := dockerCmd(ctx, "build", "-q", "-t", h.imageTag, "-f", h.Fixture.DockerfilePath(), h.Fixture.BuildContext()); err != nil { return fmt.Errorf("build %s fixture image: %w", h.Fixture.Name, err) } if _, err := dockerCmd(ctx, "network", "create", h.network); err != nil { return fmt.Errorf("create test network: %w", err) } // docker run -d prints the container id directly, so no lookup is // needed; the container name is the stable handle for later docker // calls and the id is captured for diagnostics and assertions. runArgs := []string{"run", "-d", "--name", h.container, "--network", h.network, "-p", "127.0.0.1::22"} runArgs = append(runArgs, dockerDNS()...) runArgs = append(runArgs, h.imageTag) out, err := dockerCmd(ctx, runArgs...) if err != nil { return fmt.Errorf("start %s fixture container: %w", h.Fixture.Name, err) } h.containerID = strings.TrimSpace(out) port, err := h.publishedPort(ctx) if err != nil { return err } h.port = port key, err := h.waitForSSH(ctx, h.Addr()) if err != nil { return fmt.Errorf("wait for ssh readiness on %s fixture: %w", h.Fixture.Name, err) } // Register the temp dir before writing so a write failure still // leaves it known to the cleanup defer. dir, err := os.MkdirTemp("", "rsmon-worker-harness-") if err != nil { return err } h.knownHosts = filepath.Join(dir, "known_hosts") hostsFile, err := writeKnownHosts(dir, h.Addr(), key) if err != nil { return err } h.keyPath = testKeyPath() h.knownHosts = hostsFile h.mu.Lock() h.started = true h.mu.Unlock() success = true return nil } // dockerDNS returns the `--dns` arguments to pin for fixture containers, // parsed from the comma-separated RSMON_TEST_DOCKER_DNS environment // variable. It is empty by default (Docker's embedded DNS). The override // exists so environments with flaky local resolvers can pin a reliable // upstream for the internet-facing installs (go.dev, rocketgit.ru, // proxy.golang.org), which would otherwise fail intermittently on DNS // timeouts. func dockerDNS() []string { var args []string for _, ns := range strings.Split(os.Getenv("RSMON_TEST_DOCKER_DNS"), ",") { if ns = strings.TrimSpace(ns); ns != "" { args = append(args, "--dns", ns) } } return args } // Stop releases every resource the harness created: the container, its // dedicated network, the per-instance fixture image tag (never a shared // base image), and the temp known_hosts directory. It is idempotent and // tolerates already-removed resources so teardown never fails the test // twice. func (h *Harness) Stop() error { ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) defer cancel() var errs []error if h.container != "" { if _, err := dockerCmd(ctx, "rm", "-f", h.container); err != nil && !isNotExist(err) { errs = append(errs, err) } } if h.network != "" { if _, err := dockerCmd(ctx, "network", "rm", h.network); err != nil && !isNotExist(err) { errs = append(errs, err) } } if h.imageTag != "" { if _, err := dockerCmd(ctx, "image", "rm", h.imageTag); err != nil && !isNotExist(err) && !isInUse(err) { errs = append(errs, err) } } if h.knownHosts != "" { if err := os.RemoveAll(filepath.Dir(h.knownHosts)); err != nil { errs = append(errs, fmt.Errorf("remove harness temp dir: %w", err)) } } h.mu.Lock() h.started = false h.containerID = "" h.port = 0 h.keyPath = "" h.knownHosts = "" h.mu.Unlock() return errors.Join(errs...) } // Dial connects using the same SSH library (golang.org/x/crypto/ssh) and // the same known_hosts host-key verification the installer's deploy path // relies on. The harness owns this code rather than calling into the // installer package so tests stay independent; only the library and the // verification semantics are shared. func (h *Harness) Dial() (*ssh.Client, error) { h.mu.Lock() if !h.started || h.knownHosts == "" || h.keyPath == "" { h.mu.Unlock() return nil, errors.New("harness is not started") } addr := net.JoinHostPort("127.0.0.1", strconv.Itoa(h.port)) knownHostsFile, keyPath, user := h.knownHosts, h.keyPath, h.user h.mu.Unlock() keyBytes, err := os.ReadFile(keyPath) if err != nil { return nil, err } signer, err := ssh.ParsePrivateKey(keyBytes) if err != nil { return nil, err } hostKeyCallback, err := knownhosts.New(knownHostsFile) if err != nil { return nil, err } config := &ssh.ClientConfig{ User: user, Auth: []ssh.AuthMethod{ssh.PublicKeys(signer)}, HostKeyCallback: hostKeyCallback, Timeout: 15 * time.Second, } return ssh.Dial("tcp", addr, config) } // RunCommand executes a command over an existing SSH client and returns // its stdout. Stderr is folded into the returned error message on // failure so test failures show what went wrong. func RunCommand(client *ssh.Client, command string) ([]byte, error) { session, err := client.NewSession() if err != nil { return nil, err } defer session.Close() //nolint:errcheck var stdout, stderr bytes.Buffer session.Stdout = &stdout session.Stderr = &stderr if err := session.Run(command); err != nil { if msg := strings.TrimSpace(stderr.String()); msg != "" { return stdout.Bytes(), fmt.Errorf("%w: %s", err, msg) } return stdout.Bytes(), err } return stdout.Bytes(), nil } // DockerCmd runs the Docker CLI with the given arguments, returning // stdout. Exposed so integration tests can assert teardown state. func (h *Harness) DockerCmd(ctx context.Context, args ...string) (string, error) { return dockerCmd(ctx, args...) } // Enabled reports whether Docker-backed integration tests may run. The // opt-in env var keeps ordinary `go test` and CI unit runs from pulling // or starting any container. func Enabled() bool { switch strings.ToLower(strings.TrimSpace(os.Getenv("RSMON_TEST_DOCKER"))) { case "1", "true", "yes", "on": return true } return false } // SkipUnlessEnabled skips the test with a helpful message unless the // Docker integration opt-in is set. func SkipUnlessEnabled(t interface { Helper() Skipf(format string, args ...interface{}) }, ) { t.Helper() if !Enabled() { t.Skipf("Docker/OpenSSH harness tests are opt-in; set RSMON_TEST_DOCKER=1 to run them") } } // dockerBin is the Docker CLI binary. Overridable in tests via // SetDockerBin so unit tests can stub the harness's docker calls. var dockerBin = "docker" // SetDockerBin overrides the Docker CLI binary used by the harness. // Pass an empty value to restore the default ("docker"). func SetDockerBin(name string) { if name == "" { dockerBin = "docker" return } dockerBin = name } func dockerCmd(ctx context.Context, args ...string) (string, error) { cmd := exec.CommandContext(ctx, dockerBin, args...) out, err := cmd.Output() if err != nil { if ee, ok := err.(*exec.ExitError); ok && len(ee.Stderr) > 0 { return "", fmt.Errorf("docker %s: %s", strings.Join(args, " "), strings.TrimSpace(string(ee.Stderr))) } return "", fmt.Errorf("docker %s: %w", strings.Join(args, " "), err) } return string(out), nil } // isNotExist reports whether the docker error is a missing resource // (container/network/image already removed or never created). Docker // reports missing networks and images as "... not found" and missing // containers as "No such container: ...". func isNotExist(err error) bool { if err == nil { return false } lower := strings.ToLower(err.Error()) for _, marker := range []string{"no such container", "no such network", "no such image", "is not running", "not found"} { if strings.Contains(lower, marker) { return true } } return false } // isInUse reports whether a docker image removal failed because another // container or tag still references the image. Teardown must not treat // that as a leak error: the per-instance tags make this unlikely, but // tolerating it keeps Stop deterministic under concurrent harnesses. func isInUse(err error) bool { if err == nil { return false } lower := strings.ToLower(err.Error()) for _, marker := range []string{"image is being used", "image is in use", "image is referenced"} { if strings.Contains(lower, marker) { return true } } return false } // publishedPort queries the host port docker published for the // container's port 22. func (h *Harness) publishedPort(ctx context.Context) (int, error) { out, err := dockerCmd(ctx, "port", h.container, "22/tcp") if err != nil { return 0, err } return parsePublishedPort(out) } // parsePublishedPort extracts the host port from `docker port` output // like "127.0.0.1:49153", "0.0.0.0:49153", or "::1:49153". func parsePublishedPort(out string) (int, error) { for _, line := range strings.Split(out, "\n") { line = strings.TrimSpace(line) if line == "" { continue } sep := strings.LastIndex(line, ":") if sep < 0 { continue } port, err := strconv.Atoi(strings.TrimSpace(line[sep+1:])) if err == nil && port > 0 && port <= 65535 { return port, nil } } return 0, fmt.Errorf("could not parse published port from %q", out) } // waitForSSH waits for the container's SSH listener to accept a real // handshake and returns the server host key. TCP readiness alone is not // enough; the ssh.Dial must succeed and the exec channel must answer. func (h *Harness) waitForSSH(ctx context.Context, addr string) (ssh.PublicKey, error) { if err := waitForPort(ctx, addr, 60*time.Second); err != nil { return nil, err } deadline := time.Now().Add(60 * time.Second) var lastErr error for { if ctx.Err() != nil { return nil, fmt.Errorf("ssh readiness: %w (last: %v)", ctx.Err(), lastErr) } if time.Now().After(deadline) { return nil, fmt.Errorf("ssh readiness timed out (last: %v)", lastErr) } key, err := h.handshake(addr) if err == nil { return key, nil } lastErr = err time.Sleep(500 * time.Millisecond) } } // handshake performs one full SSH exchange and captures the server host // key via the callback so the harness can write a known_hosts entry. // A fresh known_hosts file trusts the first key it sees (trust-on-first- // use); the host-key-mismatch integration test proves that a known_hosts // entry carrying a *different* key is rejected before any command runs. func (h *Harness) handshake(addr string) (ssh.PublicKey, error) { keyBytes, err := os.ReadFile(testKeyPath()) if err != nil { return nil, err } signer, err := ssh.ParsePrivateKey(keyBytes) if err != nil { return nil, err } var got ssh.PublicKey config := &ssh.ClientConfig{ User: h.user, Auth: []ssh.AuthMethod{ssh.PublicKeys(signer)}, HostKeyCallback: func(_ string, _ net.Addr, key ssh.PublicKey) error { got = key return nil }, Timeout: 5 * time.Second, } client, err := ssh.Dial("tcp", addr, config) if err != nil { return nil, err } defer client.Close() //nolint:errcheck session, err := client.NewSession() if err != nil { return nil, err } defer session.Close() //nolint:errcheck if _, err := session.Output("true"); err != nil { return nil, err } if got == nil { return nil, errors.New("no host key returned by handshake") } return got, nil } // waitForPort polls a TCP endpoint until it accepts a connection. It is // a plain network probe; SSH readiness is separately verified. func waitForPort(ctx context.Context, addr string, timeout time.Duration) error { deadline := time.Now().Add(timeout) for { if ctx.Err() != nil { return ctx.Err() } if time.Now().After(deadline) { return fmt.Errorf("port %s not open within %s", addr, timeout) } conn, err := net.DialTimeout("tcp", addr, 2*time.Second) if err == nil { _ = conn.Close() return nil } time.Sleep(500 * time.Millisecond) } } // writeKnownHosts writes a known_hosts entry for the harness address // inside an existing temp directory, using the captured server host key. func writeKnownHosts(dir, addr string, key ssh.PublicKey) (string, error) { path := filepath.Join(dir, "known_hosts") line := knownhosts.Line([]string{addr}, key) if err := os.WriteFile(path, []byte(line+"\n"), 0o600); err != nil { return "", err } return path, nil } // testKeyPath returns the shared test private key bundled with the // package. The fixtures bake the matching public key into authorized_keys. // // This keypair is strictly test-only: it grants root SSH access only to // the disposable fixture containers that bake its public key. It must // never be used for real hosts, copied into production images, or // treated as a credential anywhere outside the harness. func testKeyPath() string { return filepath.Join("testdata", "keys", "rsmon_test_ed25519") } // validateName rejects harness names that could inject shell or docker // metacharacters into container/network names. func validateName(name string) error { if name == "" || len(name) > 64 { return errors.New("harness name must be 1-64 characters") } for _, r := range name { lower := r >= 'a' && r <= 'z' digit := r >= '0' && r <= '9' if !lower && !digit && r != '-' { return fmt.Errorf("harness name %q must be lowercase alphanumeric and hyphens", name) } } return nil } func sanitize(name string) string { var b strings.Builder for _, r := range name { if r >= 'a' && r <= 'z' || r >= '0' && r <= '9' { b.WriteRune(r) } else { b.WriteByte('-') } } return b.String() }