From bd6070ee1fff8645c30ea180dcb2a0fdb9fd2b5d Mon Sep 17 00:00:00 2001 From: Gleb Tv Date: Thu, 13 Aug 2026 00:07:43 +0300 Subject: [PATCH] feat(installer): build worker source over SSH --- Makefile | 11 +- README.md | 43 +- cmd/rsmon-worker/main_test.go | 9 + cmd/rsmon-worker/management.go | 77 +++ docs/changelog.md | 82 +++ docs/implementation-roadmap.md | 17 +- docs/install.md | 8 +- docs/source-installation.md | 171 +++++- internal/installer/deploy.go | 165 ++++-- internal/installer/deploy_test.go | 12 +- internal/installer/harness/harness.go | 36 +- internal/installer/harness/harness_test.go | 15 + .../installer/harness/integration_test.go | 232 ++++++++ internal/installer/sourceinstall.go | 558 ++++++++++++++++++ internal/installer/sourceinstall_ssh_test.go | 351 +++++++++++ internal/installer/sourceinstall_test.go | 395 +++++++++++++ internal/sshinstall/plan.go | 71 ++- internal/sshinstall/plan_test.go | 37 +- 18 files changed, 2194 insertions(+), 96 deletions(-) create mode 100644 internal/installer/sourceinstall.go create mode 100644 internal/installer/sourceinstall_ssh_test.go create mode 100644 internal/installer/sourceinstall_test.go diff --git a/Makefile b/Makefile index 4760efc..d293aab 100644 --- a/Makefile +++ b/Makefile @@ -39,12 +39,15 @@ test: ./checks/llmhttp # test-ssh runs the Docker/OpenSSH source-install harness tests against -# real Alpine, Ubuntu, and Arch fixtures. Requires a working Docker -# daemon; the fixtures pull/start containers, so this is opt-in and is -# never part of the default `make test` run. +# real Alpine, Ubuntu, and Arch fixtures, including the full work +# package-3 source install (Go toolchain download, clone, and build) on +# each. Requires a working Docker daemon; the fixtures pull/start +# containers and build inside them, so this is opt-in and is never part +# of the default `make test` run. The 60m timeout covers three real +# toolchain downloads and worker builds plus the harness lifecycle tests. test-ssh: RSMON_ENV=test CWD=$(CURDIR) RSMON_TEST_DOCKER=1 go test \ - -v -timeout 30m -count=1 ./internal/installer/harness + -v -timeout 60m -count=1 ./internal/installer/harness check: go mod tidy diff --git a/README.md b/README.md index 7684802..5b36b38 100644 --- a/README.md +++ b/README.md @@ -168,21 +168,52 @@ are rejected before connecting to the remote host. The default SSH port is 22 and the default RSMon URL is `https://rsmon.ru`. Encrypted keys use `--key-passphrase-file`; password authentication uses -`--password-file`; password-protected sudo uses `--sudo-password-file`. Direct -secret flags are supported for interactive convenience but file options are -safer for automation. +`--password-file`; password-protected sudo uses `--sudo-password-file`. Strongly +prefer the file options for automation: a secret supplied through a direct flag +is visible in the process list and shell history, while a file option never +exposes it through argv. The direct flags remain available for interactive +convenience. SSH host keys are checked against `~/.ssh/known_hosts` by default. Use `--known-hosts PATH` or pin `--host-key-fingerprint SHA256:...`. The explicit `--insecure-host-key` option disables host authentication and should only be used in a trusted disposable environment. -A Go SSH source installer (remote package/toolchain/source build) is planned; -the pure detection/planning layer and the Docker/OpenSSH test harness that will -accept it are implemented. See +A Go SSH source installer is implemented as the `source-install` subcommand: it +detects the remote distro and architecture over the existing SSH transport, +installs the minimal build prerequisites, downloads and SHA-256-verifies the +pinned Go 1.26 toolchain, clones/updates the public repository, records the +resolved branch and commit, and builds the worker to a staging path. It +deliberately stops before touching the running service or its configuration +(atomic activation and rollback are the next milestone). The pure +detection/planning layer and the Docker/OpenSSH test harness that accepts it +are implemented; see [`docs/source-installation.md`](docs/source-installation.md); run the live fixture matrix with `make test-ssh`. +```bash +./bin/rsmon-worker source-install \ + --host worker.example.com \ + --user deploy \ + --identity-file ~/.ssh/id_ed25519 +``` + +By default the installer builds the remote's default branch and records what it +resolves to (the public repository currently publishes `master`). Pass +`--branch ` to pin an explicit branch; it must exist on the remote or the +install fails before building. The repository must be an `https://` URL without +userinfo. + +The built binary is left at `/opt/rsmon-worker-src/rsmon-worker` (override with +`--build-dir` / `--stage-binary`), the toolchain at `/usr/local/go` (replaced +atomically: download, verify, stage, swap with rollback), and the resolved +branch and commit in `/opt/rsmon-worker-src/rsmon-worker.commit` (written only +after a successful build). The same SSH auth, secret-file, and host-key options +as `deploy` apply; no worker token is sent because the source install does not +configure a service. Prefer `--key-passphrase-file`, `--password-file`, and +`--sudo-password-file` over their direct-flag equivalents: file options keep +secrets out of the process list and shell history. + ## Configuration | Variable | Required | Default | Purpose | diff --git a/cmd/rsmon-worker/main_test.go b/cmd/rsmon-worker/main_test.go index 5f5c3d5..3eb5b33 100644 --- a/cmd/rsmon-worker/main_test.go +++ b/cmd/rsmon-worker/main_test.go @@ -18,6 +18,15 @@ func TestDispatchManagementCommand(t *testing.T) { if handled, code := dispatchManagementCommand([]string{"install", "--help"}); !handled || code != 0 { t.Fatalf("install help = handled %t code %d", handled, code) } + if handled, code := dispatchManagementCommand([]string{"deploy", "--help"}); !handled || code != 0 { + t.Fatalf("deploy help = handled %t code %d", handled, code) + } + if handled, code := dispatchManagementCommand([]string{"source-install", "--help"}); !handled || code != 0 { + t.Fatalf("source-install help = handled %t code %d", handled, code) + } + if handled, code := dispatchManagementCommand([]string{"source-install", "--identity-file", t.TempDir() + "/missing"}); !handled || code == 0 { + t.Fatalf("source-install without host/user = handled %t code %d", handled, code) + } } func TestSecretValue(t *testing.T) { diff --git a/cmd/rsmon-worker/management.go b/cmd/rsmon-worker/management.go index 340bc3d..870c6d5 100644 --- a/cmd/rsmon-worker/management.go +++ b/cmd/rsmon-worker/management.go @@ -19,6 +19,8 @@ func dispatchManagementCommand(args []string) (bool, int) { return true, installCommand(args[1:]) case "deploy": return true, deployCommand(args[1:]) + case "source-install": + return true, sourceInstallCommand(args[1:]) default: return false, 0 } @@ -153,3 +155,78 @@ func secretValue(direct, path string) (string, error) { } return strings.TrimRight(string(b), "\r\n"), nil } + +// sourceInstallCommand drives the remote source installer (work package +// 3 of docs/source-installation.md): prerequisites, verified Go +// toolchain, clone/update, resolved branch/commit record, and a staging +// build. It reuses the deploy SSH options; it never touches the running +// service, configuration, or data directory on the remote host. +func sourceInstallCommand(args []string) int { + fs := flag.NewFlagSet("source-install", flag.ContinueOnError) + fs.SetOutput(os.Stderr) + var opts installer.SourceInstallOptions + var passphraseFile, passwordFile, sudoPasswordFile string + fs.StringVar(&opts.Host, "host", "", "SSH server hostname or address") + fs.IntVar(&opts.Port, "port", 22, "SSH server port") + fs.StringVar(&opts.User, "user", "", "SSH username") + fs.StringVar(&opts.IdentityFile, "identity-file", "", "SSH private key path") + fs.StringVar(&opts.KeyPassphrase, "key-passphrase", "", "SSH private key passphrase") + fs.StringVar(&passphraseFile, "key-passphrase-file", "", "file containing the private key passphrase") + fs.StringVar(&opts.Password, "password", "", "SSH login password") + fs.StringVar(&passwordFile, "password-file", "", "file containing the SSH login password") + fs.StringVar(&opts.SudoPassword, "sudo-password", "", "remote sudo password") + fs.StringVar(&sudoPasswordFile, "sudo-password-file", "", "file containing the remote sudo password") + fs.StringVar(&opts.KnownHostsFile, "known-hosts", "", "known_hosts path (default: ~/.ssh/known_hosts)") + fs.StringVar(&opts.HostKeyFingerprint, "host-key-fingerprint", "", "expected SHA256 SSH host-key fingerprint") + fs.BoolVar(&opts.InsecureHostKey, "insecure-host-key", false, "disable SSH host-key verification (unsafe)") + fs.StringVar(&opts.Repo, "repo", "", "worker repository to clone/update (default: public rocketgit.ru repo)") + fs.StringVar(&opts.Branch, "branch", "", "branch to build; empty resolves the remote default branch") + fs.StringVar(&opts.GoVersion, "go-version", "", "Go toolchain version (default: pinned 1.26.0)") + fs.StringVar(&opts.GoArch, "go-arch", "", "Go download archive suffix; empty derives it from the remote architecture") + fs.StringVar(&opts.BuildDir, "build-dir", "", "remote clone/build directory (default /opt/rsmon-worker-src)") + fs.StringVar(&opts.GoModuleProxy, "go-proxy", "", "GOPROXY for the remote build (default: Go default)") + fs.StringVar(&opts.ToolchainDir, "toolchain-dir", "", "remote Go install path ending in /go (default /usr/local/go)") + fs.StringVar(&opts.StageBinary, "stage-binary", "", "staging binary path (default /rsmon-worker)") + fs.DurationVar(&opts.SessionTimeout, "session-timeout", 0, "per-remote-command timeout (default 30m; 0 uses the default)") + fs.Usage = func() { + fmt.Fprintln(fs.Output(), "Usage: rsmon-worker source-install --host HOST --user USER [SSH options] [source options]") + fs.PrintDefaults() + } + if err := fs.Parse(args); err != nil { + if errors.Is(err, flag.ErrHelp) { + return 0 + } + return 2 + } + if fs.NArg() != 0 { + fs.Usage() + return 2 + } + var err error + if opts.KeyPassphrase, err = secretValue(opts.KeyPassphrase, passphraseFile); err != nil { + fmt.Fprintf(os.Stderr, "source-install failed: %v\n", err) + return 1 + } + if opts.Password, err = secretValue(opts.Password, passwordFile); err != nil { + fmt.Fprintf(os.Stderr, "source-install failed: %v\n", err) + return 1 + } + if opts.SudoPassword, err = secretValue(opts.SudoPassword, sudoPasswordFile); err != nil { + fmt.Fprintf(os.Stderr, "source-install failed: %v\n", err) + return 1 + } + res, err := installer.SourceInstall(opts) + if err != nil { + fmt.Fprintf(os.Stderr, "source-install failed: %v\n", err) + return 1 + } + fmt.Printf("source install complete\n") + fmt.Printf(" distro: %s\n", res.Detection.Summarize()) + fmt.Printf(" toolchain: %s (%s) at %s\n", res.Plan.Toolchain.Version, res.GoArch, res.ToolchainDir) + fmt.Printf(" branch: %s\n", res.ResolvedBranch) + fmt.Printf(" commit: %s\n", res.ResolvedCommit) + fmt.Printf(" record file: %s\n", res.RecordFile) + fmt.Printf(" staged build: %s\n", res.StageBinary) + fmt.Println(" status: not installed as a service (activation is the next milestone)") + return 0 +} diff --git a/docs/changelog.md b/docs/changelog.md index f9df91f..57eff81 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -2,6 +2,88 @@ ## 2026-08-12 +### Source-install hardening review + +- Fail-closed remote scripts: checkout, branch resolution, and build steps now + run under `set -eu` (and package/record steps chain with `&&`), so a failed + checkout or fetch can never be masked by a stale `rev-parse` or subsequent + command. The checkout step additionally refuses (`git diff --quiet` / + `--cached --quiet`) before the destructive `checkout -B`, because `-B` + silently discards local changes and would otherwise never fail on a dirty + tree. A dirty-tree checkout failure surfaces as a `check out branch` error + before the build runs; the new `TestSourceInstallSSHCheckoutFailureNotMasked` + unit test and `TestSourceInstallDirtyCheckoutPreservesStaging` Docker test + prove the previous staging binary and commit record are preserved + byte-for-byte. +- Atomic toolchain replacement: the Go toolchain is downloaded, SHA-256 + verified, extracted into a same-filesystem staging dir, verified to report + the target version, and only then swapped into `ToolchainDir` with the prior + toolchain moved to a sibling `.go-backup` that is restored on swap failure. + A failed download/verify/extract/swap never destroys the prior Go. +- Record-after-build pairing: `rsmon-worker.commit` is written only after a + successful build, so the record and the staged binary always correspond to + the same commit. The build verifies `.new --version` before an atomic + `mv -f` over the previous staging binary; `GOMODCACHE` is now set alongside + `GOCACHE` inside the build dir so reruns reuse both caches. +- Origin verification: an existing checkout's `remote.origin.url` must exactly + match the configured repository before anything is fetched or built. +- Repository hardening: only `https://` clone URLs without userinfo are + accepted (`ValidateRepoURL`, enforced before dialing and again when + planning). +- Explicit charset validation for Go version and architecture overrides + (`sshinstall.ValidGoVersion` / `ValidGoArch`) before any remote mutation. +- Bounded remote execution: each remote command is capped by + `--session-timeout` (default 30m) and captured stdout is size-bounded + alongside the existing stderr bound; deploy's streaming `runRemote` keeps its + historical no-timeout behavior. +- The source installer now defaults to the remote's default branch (the public + repo publishes `master`) instead of the plan's stale `main` default, while + `--branch` still pins an explicit branch that must exist remotely. The + README quickstart no longer shows the incorrect `--branch main`. +- CLI secret flags keep their compatibility, but docs now explicitly state that + file options (`-password-file`, etc.) keep secrets out of argv and shell + history while direct flags expose them through the process list. +- The harness accepts `RSMON_TEST_DOCKER_DNS` (comma-separated) to pin + `docker run --dns` for fixture containers, so internet-facing installs are + not at the mercy of a flaky local resolver. + +### Remote source-install execution (work package 3) + +- Added `installer.SourceInstall` (`internal/installer/sourceinstall.go`): + executes the source-install flow through the existing SSH transport, + reusing the `deploy` command's `SSHOptions` (keys, passphrases, + passwords, sudo passwords, known-hosts, pinned fingerprints) and its + privilege path. Extracted the shared `SSHOptions` struct and a + `sudoWrap` helper so deploy and source install cannot diverge. +- Steps implemented: minimal package-prerequisite install per distro + (`apk`/`apt`/`pacman`/`dnf`, never a compiler), SHA-256-verified Go 1.26 + toolchain download/extraction with an idempotent version-skip and temp-dir + cleanup, clone-or-update of the public repository (with a bounded 3-attempt + retry for transient DNS/TLS/proxy failures), resolution of the remote + default branch (a pinned branch must exist remotely), a resolved branch and + commit record at `/rsmon-worker.commit`, and a staging build + (`CGO_ENABLED=0`, `-trimpath`, repository `-ldflags`) verified via + `--version`. The running service, config, and data directory are untouched + (work package 4 boundary). +- Security: every interpolated remote value is single-quoted; branch and + commit values are strictly validated; no worker token or control-plane + credential is sent; sudo passwords travel only over session stdin; remote + errors are bounded (stderr truncated in `runRemoteOutput`). +- Added unit tests for the remote scripts, option validation, branch/commit + parsing, sudo wrapping, the secrets-absent contract, and an in-process + real-SSH orchestration flow (with missing-pinned-branch, build-failure, and + detection-failure paths). +- Added `TestSourceInstallFixtures` to the Docker/OpenSSH harness: each of + Alpine, Ubuntu, and Arch installs from a clean state through the real + harness transport (prerequisite install, verified Go 1.26, clone, resolved + commit, staging build), then a rerun proves idempotency (same branch, + toolchain reuse, no temp leaks). All three resolved the public repo's + `master` at `4651deb2...` in the recorded run. `make test-ssh` timeout + raised to 60m. +- Documented the branch-resolution reality: the public repository currently + publishes `master`, and the installer records whatever the remote default + branch resolves to. + ### Source-install harness and planning foundations (work packages 1-2) - Added `internal/installer/harness`: a reusable Docker/OpenSSH test harness diff --git a/docs/implementation-roadmap.md b/docs/implementation-roadmap.md index af1c074..e1428a4 100644 --- a/docs/implementation-roadmap.md +++ b/docs/implementation-roadmap.md @@ -15,8 +15,8 @@ Worker repository: metadata; - add a package/install smoke test for Docker and systemd artifacts; - add the Docker/OpenSSH source-install matrix for Alpine, Ubuntu, and Arch from - [source-installation.md](source-installation.md), using Go 1.26 and branch - `main`; + [source-installation.md](source-installation.md), using Go 1.26 and the + remote default branch; - document immutable SHA and release tags as production defaults. Source-install foundations landed: @@ -26,9 +26,16 @@ Source-install foundations landed: reliable teardown, gated behind `RSMON_TEST_DOCKER` (`make test-ssh`); - [x] pure distro/toolchain/source-install planning (`internal/sshinstall`): os-release detection, package-manager/init resolution, pinned Go 1.26 - toolchain with published SHA-256, and a plan the executor will run; -- [ ] remote package install, Go download, clone, and build execution through - the SSH transport (source-install work package 3); + toolchain with published SHA-256, and a plan the executor runs; +- [x] remote package install, Go download, clone, and build execution through + the SSH transport (`installer.SourceInstall`): prerequisite install, + SHA-256-verified Go toolchain, clone/update of the public repo, resolved + branch/commit record, and a staging build. Running service/config is not + touched (source-install work package 3); +- [ ] atomic service activation, rollback, and failure-preservation tests over + SSH (source-install work package 4); +- [ ] run the full source-install E2E matrix in CI (source-install work + package 5). Gate: a push publishes `sha-<12>` and `latest` manifests for both platforms, and a container remains healthy when the control plane is unavailable. diff --git a/docs/install.md b/docs/install.md index 8b0ccd6..c9add84 100644 --- a/docs/install.md +++ b/docs/install.md @@ -5,8 +5,12 @@ Docker image) into a running, enabled systemd service on a Linux host. It is the supported way to deploy the worker: it writes the configuration, the systemd unit, and the data directory, then starts the service. -> A Go SSH source installer is planned in -> [source-installation.md](source-installation.md). Today +> A Go SSH source installer (`rsmon-worker source-install`) builds the worker +> from source on a remote host over SSH - prerequisites, verified Go toolchain, +> clone/update of the public repository (resolved to the remote default branch +> unless pinned), resolved branch/commit record, and a staging build - without +> yet installing a service. See +> [source-installation.md](source-installation.md). Today the local > `install` copies the binary you invoke it from (or pulls the `--image` > digest), so build first with `make build` and run the resulting > `./bin/rsmon-worker`. diff --git a/docs/source-installation.md b/docs/source-installation.md index 872e775..2e48f64 100644 --- a/docs/source-installation.md +++ b/docs/source-installation.md @@ -3,8 +3,9 @@ ## Status In progress. Work package 1 (reusable Docker/OpenSSH harness and distro -fixtures) and the pure detection/planning foundation (work package 2 -core) are implemented: +fixtures), the pure detection/planning foundation (work package 2 core), +and work package 3 (remote execution through the existing SSH transport) +are implemented: - `internal/installer/harness` builds and runs real OpenSSH containers for Alpine, Ubuntu, and Arch, waits for real network readiness, captures @@ -17,13 +18,23 @@ core) are implemented: and init system from `/etc/os-release`, plans the pinned Go 1.26 toolchain (published SHA-256) for the remote architecture, and produces a pure source-install plan. It executes nothing. +- `installer.SourceInstall` (work package 3) executes the plan through + the same SSH transport, authentication, and host-key verification the + `deploy` command uses. It installs the minimal package prerequisites, + downloads and SHA-256-verifies the pinned Go toolchain before + extraction, clones/updates the public repository, checks out the + resolved branch, records the resolved branch and commit, and builds the + worker to a staging path. It deliberately does not install or replace + the running service, configuration, or data directory: atomic + activation and rollback are the next work package. The current Go installer can still only upload a binary or deploy an -immutable Docker image over SSH. Remote package/toolchain/source build -execution (work packages 3 and 4) is not implemented yet; the acceptance -test currently stops after detection, clean-state, and planning -assertions. Existing tests are unit tests; the harness tests run against -live OpenSSH containers when explicitly enabled. +immutable Docker image over SSH. Source installs now build remotely to a +staging path, but service activation over SSH (work package 4) is not +implemented yet; the acceptance test stops after install-to-staging and +idempotent-rerun assertions. Existing tests are unit tests plus the +harness tests that run against live OpenSSH containers when explicitly +enabled. ## Initial Platform Scope @@ -49,7 +60,8 @@ It then: 3. downloads the pinned Go 1.26 toolchain for the detected architecture and verifies the published SHA-256; 4. clones `https://rocketgit.ru/rsmon/worker.git` or updates an existing clone; -5. checks out branch `main` and records the resolved commit; +5. checks out the resolved branch (the pinned branch when one is configured, + otherwise the remote's default branch) and records the resolved commit; 6. builds a reproducible worker binary with the repository build flags; 7. atomically installs the binary, validated environment, data directory, and service definition; @@ -59,6 +71,100 @@ Repository, branch, Go version, checksum source, build directory, and Go module proxy may be configurable, but production output records their resolved values. The default repository is publicly readable and requires no source credential. +## Work Package 3: Remote Execution To A Staging Path + +Work package 3 is `installer.SourceInstall` in +`internal/installer/sourceinstall.go`. It reuses the `deploy` command's +`SSHOptions` (authentication, sudo password, known-hosts and fingerprint +verification) and runs every remote step with the same privilege path +(root, passwordless sudo, or `sudo -S -p ''` with the password delivered +only over stdin). Steps 1-6 of the flow above are implemented; step 7 +(atomic install) is deliberately the next work package. + +Per step: + +- **Prerequisite install.** `packageScript` renders the distro's + idempotent command (`apk add --no-cache`, `apt-get update` + + `apt-get install -y --no-install-recommends`, `pacman -Sy --noconfirm + --needed`, `dnf install -y`) for the minimal plan packages (`git`, + `ca-certificates`, `curl`, `tar`, `gzip`). No compiler is ever planned + or installed. +- **Toolchain.** `toolchainScript` downloads the pinned Go tarball into a + `mktemp` temp dir, verifies it with `sha256sum -c -` *before* + extraction, extracts into a staging dir on the same filesystem as + `ToolchainDir` (which must end in `/go`, default `/usr/local/go`), + verifies the staged toolchain reports the target version, and only then + swaps it into place. The prior toolchain is moved to a sibling + `.go-backup` and is restored if the swap fails, so a failed + download/verify/extract/swap always leaves the prior Go untouched. A + present toolchain that already reports the target version is reused, so + reruns do not re-download. Temp, staging, and backup directories are + removed on success and failure. +- **Clone/update.** `cloneUpdateScript` clones the repository when + `BuildDir` has no `.git` and otherwise fetches with `--prune`, so a + rerun updates in place. An existing checkout's `remote.origin.url` must + exactly match the configured repository before anything is fetched or + built, so the installer can never fetch or build an unconfigured + repository. The clone/fetch retries up to three times (2s apart) + because real repositories can be transiently unreachable (DNS, TLS, or + proxy hiccups); three bounded attempts keep a momentary outage from + failing a full source install. +- **Resolved branch.** The installer resolves the remote default branch + via `git remote set-head origin --auto` + + `git symbolic-ref --short refs/remotes/origin/HEAD`. When no branch is + pinned it builds the remote default (the public repo currently + publishes `master`); a pinned branch must exist remotely or the install + fails before the build. The resolved branch and the `git rev-parse + HEAD` commit (validated as 40 lowercase hex) are returned by + `SourceInstall`. +- **Staging build.** `buildScript` builds with `CGO_ENABLED=0`, + `-trimpath`, the repository's own `-ldflags` shape (version `dev`, + resolved commit short form, UTC build date), and both `GOCACHE` and + `GOMODCACHE` inside the build dir (so reruns reuse them), plus an + optional `GOPROXY`. The binary is built to a sibling `.new`, + verified with `.new --version`, and only then atomically swapped + over `/rsmon-worker` (or `StageBinary`), so a failed build + never replaces the previous staging binary. It is not written to + `/usr/local/bin`. +- **Commit record.** The `rsmon-worker.commit` record (a + `branch=...` / `commit=...` format in the build dir) is written only + *after* a successful build, so the record and the staged binary always + correspond to the same commit. + +Security properties of work package 3: + +- Every interpolated value (repository, branch, build dir, URLs, SHA-256, + package names, paths) is single-quoted; repository, branch, commit, Go + version, and Go architecture values are additionally validated with + strict patterns. No worker token or control-plane credential is ever + sent: the install stages a binary and touches no service configuration. +- The repository must be an `https://` URL without userinfo, so source + credentials cannot reach the remote clone command or the clone's + config. +- Sudo passwords are delivered over the session's stdin only, never in a + command string (the same `sudoWrap` path the `deploy` command uses). + Direct `--password`/`--sudo-password`/`--key-passphrase` flags remain + available but expose the value through the process list and shell + history; the CLI docs strongly prefer the `-file` variants. The source + installer sends no worker token at all. +- Every step script fails closed: `set -eu` (or an explicit retry that + exits non-zero) is used, so a failed checkout or fetch can never be + masked by a stale subsequent command. The checkout step refuses before + the destructive `checkout -B` when the tracked working tree is dirty + (`git diff --quiet` / `--cached --quiet`), because `checkout -B` would + silently discard local changes; a dirty-tree rerun fails at checkout and + leaves the previous staging binary and commit record untouched. +- Remote errors are bounded: each step returns a step-labelled error, + stderr and captured stdout are size-bounded in `runRemoteOutput`, and + each remote command is capped by `--session-timeout` (default 30m). +- Toolchain temp, staging, and backup directories are removed on success + and failure, and the acceptance test asserts no `/tmp/rsmon-toolchain-*` + or `/usr/local/.go-staging-*`/`.go-backup` leaks after the rerun. +- Failed builds and failed checkouts leave the previous staging binary + untouched (the binary is only overwritten by an atomically-swapped + successful build, and a checkout failure aborts before the build); + there is no running service or configuration to preserve yet. + ## Docker OpenSSH Test Harness Adapt the real-network pattern from `/data/_swap/sshkeymanager`: start an @@ -93,6 +199,11 @@ harness proves is that a known_hosts entry carrying a *different* key is rejected before any command runs (the `TestHarnessHostKeyMismatch` test), not that a fingerprint is pinned. +The fixtures install over the public internet, so environments with flaky +local resolvers can pin a reliable upstream via the comma-separated +`RSMON_TEST_DOCKER_DNS` variable (applied as `docker run --dns ...`); it +is empty by default, keeping Docker's embedded DNS. + Teardown (`docker rm -f` + `docker network rm` + per-instance `docker image rm` + temp-dir removal) is idempotent, runs on every `Start` error path, and is verified by a dedicated test. Each harness @@ -158,15 +269,26 @@ RSMON_TEST_DOCKER=1 go test -v -count=1 -timeout 30m ./internal/installer/harnes ## Idempotency And Security -- A second run updates/fetches safely and leaves one active service. +- A second run updates/fetches safely: the toolchain is reused when the version + matches, the clone's origin is verified against the configured repository and + then fetched in place, the resolved branch/commit record is rewritten after + the new build succeeds, and the staging build atomically swaps over the + previous staging binary. The "one active service" guarantee is a work package + 4 property (activation); work package 3 leaves no running service to + duplicate and no leaked toolchain temp files. - Wrong host fingerprints fail before remote mutation. The harness's fresh known_hosts file is trust-on-first-use; its dedicated mismatch test dials against a known_hosts entry carrying a different server key and proves the dial fails before any command runs. -- Tokens/passwords come from files or stdin-safe channels and never appear in - command arguments, logs, source checkout, or shell history. +- Tokens/passwords should come from files: `--key-passphrase-file`, + `--password-file`, and `--sudo-password-file` keep secrets out of argv and + shell history, while the equivalent direct flags expose them through the + process list. The source install itself sends no worker token or + control-plane credential at all, and sudo passwords travel only over the + session's stdin. - Remote temporary files are removed on success and failure. -- Failed builds do not replace a working binary or service definition. +- Failed builds, failed checkouts, and failed toolchain swaps do not replace a + working binary or service definition. - Package-manager and download failures return bounded actionable errors. - The installer verifies Go tarball checksum before extraction. @@ -175,7 +297,9 @@ RSMON_TEST_DOCKER=1 go test -v -count=1 -timeout 30m ./internal/installer/harnes - [x] 1. Add reusable Docker/OpenSSH harness and distro fixtures. - [x] 2. Add pure distro/toolchain/source-install script planning and unit tests (detection + planning foundation; remote execution is work package 3). -- [ ] 3. Execute source installation through the existing SSH transport. +- [x] 3. Execute source installation through the existing SSH transport + (prerequisites, verified Go toolchain, clone/update, resolved branch and + commit record, and a build to a staging path; no service activation). - [ ] 4. Add atomic build/install, idempotency, and failure rollback. - [ ] 5. Add Alpine, Ubuntu, and Arch network E2E tests to CI. - [ ] 6. Add CentOS-family support. @@ -184,13 +308,16 @@ RSMON_TEST_DOCKER=1 go test -v -count=1 -timeout 30m ./internal/installer/harnes ## Acceptance Gates - [ ] All three initial Linux images install from a clean state through OpenSSH. + Work package 3 covers the install-to-staging half (package install, + verified Go 1.26, clone/update, resolved commit, staging build); the + service-start half is the work package 4 gate. - [ ] The built worker reports the expected version/commit and serves `/healthz`. - [ ] Re-running the installer succeeds without duplicate services or leaked files. - [ ] Host-key, checksum, clone, build, and service-start failure tests preserve the previous installation. - [ ] CI uses approved registry mirrors and cleans every test container/network. -## Verified Test Evidence (work packages 1 and 2) +## Verified Test Evidence (work packages 1-3) Recorded 2026-08-12 from `make test-ssh` (Docker Engine 29.7.1): @@ -202,3 +329,19 @@ Recorded 2026-08-12 from `make test-ssh` (Docker Engine 29.7.1): - Host-key mismatch, host-key stability, failed-start cleanup, and complete teardown (container, network, fixture image tag, and temp dir gone) tests pass; no test container, network, or image tag is left behind. +- `TestSourceInstallFixtures` runs the full work-package-3 flow on each + fixture from a clean state over the harness's real OpenSSH transport: + prerequisite install, verified Go 1.26 toolchain download/extraction, clone + of the public repository, resolution of its default branch, resolved-commit + record file, and a staging build that reports the resolved commit via + `--version`. The public repo's `HEAD` was `master` at `4651deb2...` during + the run, and all three fixtures resolved to that branch and commit (the + installer records whatever the remote publishes). A rerun succeeds, keeps + the same branch, reuses the toolchain, and leaves no + `/tmp/rsmon-toolchain-*` temp dirs. The running service and its + configuration are not touched (work package 4 boundary). +- The unit suite covers the per-step remote scripts, option validation, + branch/commit parsing, sudo wrapping (password never in the command), the + secrets-absent contract, and an in-process real-SSH orchestration flow with + failure paths for missing pinned branches, build failures, and detection + failures. diff --git a/internal/installer/deploy.go b/internal/installer/deploy.go index c7c0d1e..a2fa6e7 100644 --- a/internal/installer/deploy.go +++ b/internal/installer/deploy.go @@ -18,7 +18,14 @@ import ( "golang.org/x/crypto/ssh/knownhosts" ) -type DeployOptions struct { +// SSHOptions is the SSH connection and authentication surface shared by +// the deploy command and the source installer. Secrets (passwords, +// passphrases, sudo passwords) can be supplied through direct flags or +// file options. The CLI layer should strongly prefer file options: when +// read from a file they never appear in command arguments, logs, or +// shell history, while direct flags expose the value through the process +// list and shell history. +type SSHOptions struct { Host string Port int User string @@ -29,12 +36,16 @@ type DeployOptions struct { KnownHostsFile string HostKeyFingerprint string InsecureHostKey bool - Binary string - Token string - URL string - Docker bool - Image string - NoStart bool +} + +type DeployOptions struct { + SSHOptions + Binary string + Token string + URL string + Docker bool + Image string + NoStart bool } func Deploy(opts DeployOptions) error { @@ -71,11 +82,11 @@ func Deploy(opts DeployOptions) error { return err } } - auth, err := sshAuth(opts) + auth, err := sshAuth(opts.SSHOptions) if err != nil { return err } - hostKey, err := hostKeyCallback(opts) + hostKey, err := hostKeyCallback(opts.SSHOptions) if err != nil { return err } @@ -111,16 +122,7 @@ func Deploy(opts DeployOptions) error { if opts.NoStart { args += " --no-start" } - var command string - var stdin []byte - if opts.User == "root" { - command = args - } else if opts.SudoPassword != "" { - command = "sudo -S -p '' -- " + args - stdin = []byte(opts.SudoPassword + "\n") - } else { - command = "sudo -n -- " + args - } + command, stdin := sudoWrap(opts.User, opts.SudoPassword, args) if err := runRemote(client, command, stdin); err != nil { return fmt.Errorf("remote install: %w", err) } @@ -137,23 +139,30 @@ func deployDocker(client *ssh.Client, opts DeployOptions, remoteEnv, remoteUnit if !opts.NoStart { script += " && systemctl restart rsmon-worker.service && systemctl is-active --quiet rsmon-worker.service" } - command := "sh -c " + shellQuote(script) - var stdin []byte - if opts.User != "root" { - if opts.SudoPassword != "" { - command = "sudo -S -p '' -- " + command - stdin = []byte(opts.SudoPassword + "\n") - } else { - command = "sudo -n -- " + command - } - } + command, stdin := sudoWrap(opts.User, opts.SudoPassword, "sh -c "+shellQuote(script)) if err := runRemote(client, command, stdin); err != nil { return fmt.Errorf("remote Docker install: %w", err) } return nil } -func sshAuth(opts DeployOptions) ([]ssh.AuthMethod, error) { +// sudoWrap prefixes a remote command with the privilege path required to +// run it as root: the plain command for a root user, `sudo -n` when the +// user has passwordless sudo, and `sudo -S` with an empty prompt when a +// sudo password is configured. The sudo password is delivered only over +// the session's stdin, never in the command string, so it cannot appear +// in process listings, logs, or shell history. +func sudoWrap(user, sudoPassword, command string) (string, []byte) { + if user == "root" { + return command, nil + } + if sudoPassword != "" { + return "sudo -S -p '' -- " + command, []byte(sudoPassword + "\n") + } + return "sudo -n -- " + command, nil +} + +func sshAuth(opts SSHOptions) ([]ssh.AuthMethod, error) { var methods []ssh.AuthMethod if opts.IdentityFile != "" { key, err := os.ReadFile(opts.IdentityFile) @@ -180,7 +189,7 @@ func sshAuth(opts DeployOptions) ([]ssh.AuthMethod, error) { return methods, nil } -func hostKeyCallback(opts DeployOptions) (ssh.HostKeyCallback, error) { +func hostKeyCallback(opts SSHOptions) (ssh.HostKeyCallback, error) { if opts.HostKeyFingerprint != "" { want := opts.HostKeyFingerprint return func(_ string, _ net.Addr, key ssh.PublicKey) error { @@ -242,6 +251,79 @@ func uploadReader(client *ssh.Client, src io.Reader, remotePath string, mode os. } func runRemote(client *ssh.Client, command string, stdin []byte) error { + // Deploy commands are short and stream to the terminal; they keep the + // historical behavior with no timeout. + return runRemoteWithTimeout(client, command, stdin, os.Stdout, os.Stderr, 0) +} + +// maxRemoteError bounds the stderr snippet folded into runRemoteOutput +// errors so a verbose remote failure cannot produce an unbounded error +// string. +const maxRemoteError = 4096 + +// maxRemoteOutput bounds the stdout captured by runRemoteOutput so a +// noisy remote command cannot exhaust memory. +const maxRemoteOutput = 1 << 20 // 1 MiB + +// boundedBuffer is an io.Writer that silently discards everything past +// max bytes and remembers whether truncation happened. +type boundedBuffer struct { + buf bytes.Buffer + max int + truncated bool +} + +func (b *boundedBuffer) Write(p []byte) (int, error) { + if len(p) == 0 { + return 0, nil + } + if b.buf.Len() >= b.max { + b.truncated = true + return len(p), nil + } + remaining := b.max - b.buf.Len() + if len(p) > remaining { + b.buf.Write(p[:remaining]) + b.truncated = true + return len(p), nil + } + return b.buf.Write(p) +} + +func (b *boundedBuffer) Bytes() []byte { return b.buf.Bytes() } + +func (b *boundedBuffer) String() string { return b.buf.String() } + +// runRemoteOutput executes a remote command and returns its captured, +// size-bounded stdout. Stderr is folded into the returned error on +// failure (bounded to maxRemoteError bytes) so operators see what went +// wrong without a bounded failure dumping unbounded output. The command +// is aborted if it outlives timeout (<= 0 disables the timeout). +func runRemoteOutput(client *ssh.Client, command string, stdin []byte, timeout time.Duration) ([]byte, error) { + var stdout, stderr boundedBuffer + stdout.max = maxRemoteOutput + stderr.max = maxRemoteError + if err := runRemoteWithTimeout(client, command, stdin, &stdout, &stderr, timeout); err != nil { + msg := strings.TrimSpace(stderr.String()) + if msg != "" { + if stderr.truncated { + msg += "..." + } + return stdout.Bytes(), fmt.Errorf("%w: %s", err, msg) + } + return stdout.Bytes(), err + } + out := stdout.Bytes() + if stdout.truncated { + out = append(out, []byte("\n...[output truncated]")...) + } + return out, nil +} + +// runRemoteWithTimeout runs a remote command, optionally aborting it +// when it outlives timeout (<= 0 disables the timeout). The session is +// closed and the blocked Run is unblocked when the timer fires. +func runRemoteWithTimeout(client *ssh.Client, command string, stdin []byte, stdout, stderr io.Writer, timeout time.Duration) error { session, err := client.NewSession() if err != nil { return err @@ -250,9 +332,24 @@ func runRemote(client *ssh.Client, command string, stdin []byte) error { if stdin != nil { session.Stdin = bytes.NewReader(stdin) } - session.Stdout = os.Stdout - session.Stderr = os.Stderr - return session.Run(command) + session.Stdout = stdout + session.Stderr = stderr + + if timeout <= 0 { + return session.Run(command) + } + done := make(chan error, 1) + go func() { done <- session.Run(command) }() + timer := time.NewTimer(timeout) + defer timer.Stop() + select { + case err := <-done: + return err + case <-timer.C: + _ = session.Close() // abort the remote command and free the session + <-done + return fmt.Errorf("remote command timed out after %s", timeout) + } } func shellQuote(value string) string { diff --git a/internal/installer/deploy_test.go b/internal/installer/deploy_test.go index cee0bd2..21b8d97 100644 --- a/internal/installer/deploy_test.go +++ b/internal/installer/deploy_test.go @@ -19,14 +19,14 @@ func TestFingerprintHostKeyCallback(t *testing.T) { if err != nil { t.Fatal(err) } - callback, err := hostKeyCallback(DeployOptions{HostKeyFingerprint: ssh.FingerprintSHA256(publicKey)}) + callback, err := hostKeyCallback(SSHOptions{HostKeyFingerprint: ssh.FingerprintSHA256(publicKey)}) if err != nil { t.Fatal(err) } if err := callback("host", &net.TCPAddr{}, publicKey); err != nil { t.Fatalf("matching fingerprint rejected: %v", err) } - callback, err = hostKeyCallback(DeployOptions{HostKeyFingerprint: "SHA256:wrong"}) + callback, err = hostKeyCallback(SSHOptions{HostKeyFingerprint: "SHA256:wrong"}) if err != nil { t.Fatal(err) } @@ -36,15 +36,17 @@ func TestFingerprintHostKeyCallback(t *testing.T) { } func TestKnownHostsMissingFile(t *testing.T) { - if _, err := hostKeyCallback(DeployOptions{KnownHostsFile: t.TempDir() + "/missing"}); err == nil { + if _, err := hostKeyCallback(SSHOptions{KnownHostsFile: t.TempDir() + "/missing"}); err == nil { t.Fatal("missing known_hosts file accepted") } } func TestDeployRejectsMutableDockerImageBeforeConnecting(t *testing.T) { err := Deploy(DeployOptions{ - Host: "unreachable.example.test", - User: "deploy", + SSHOptions: SSHOptions{ + Host: "unreachable.example.test", + User: "deploy", + }, Token: "token", Docker: true, Image: "reg.rsxx.ru/rsmon/rsmon-worker:latest", diff --git a/internal/installer/harness/harness.go b/internal/installer/harness/harness.go index 51b09b8..55f7440 100644 --- a/internal/installer/harness/harness.go +++ b/internal/installer/harness/harness.go @@ -168,6 +168,14 @@ func (h *Harness) Addr() string { return net.JoinHostPort("127.0.0.1", strconv.Itoa(h.port)) } +// Port returns the published host port of the container's SSH listener +// (the host is always 127.0.0.1). 0 before Start. +func (h *Harness) Port() int { + h.mu.Lock() + defer h.mu.Unlock() + return h.port +} + // Start builds the fixture image, starts the container, waits for real // SSH readiness, captures the server host key into a temp known_hosts // file, and records the published port. Every error path cleans up the @@ -201,13 +209,10 @@ func (h *Harness) Start(ctx context.Context) error { // docker run -d prints the container id directly, so no lookup is // needed; the container name is the stable handle for later docker // calls and the id is captured for diagnostics and assertions. - out, err := dockerCmd( - ctx, "run", "-d", - "--name", h.container, - "--network", h.network, - "-p", "127.0.0.1::22", - h.imageTag, - ) + runArgs := []string{"run", "-d", "--name", h.container, "--network", h.network, "-p", "127.0.0.1::22"} + runArgs = append(runArgs, dockerDNS()...) + runArgs = append(runArgs, h.imageTag) + out, err := dockerCmd(ctx, runArgs...) if err != nil { return fmt.Errorf("start %s fixture container: %w", h.Fixture.Name, err) } @@ -244,6 +249,23 @@ func (h *Harness) Start(ctx context.Context) error { return nil } +// dockerDNS returns the `--dns` arguments to pin for fixture containers, +// parsed from the comma-separated RSMON_TEST_DOCKER_DNS environment +// variable. It is empty by default (Docker's embedded DNS). The override +// exists so environments with flaky local resolvers can pin a reliable +// upstream for the internet-facing installs (go.dev, rocketgit.ru, +// proxy.golang.org), which would otherwise fail intermittently on DNS +// timeouts. +func dockerDNS() []string { + var args []string + for _, ns := range strings.Split(os.Getenv("RSMON_TEST_DOCKER_DNS"), ",") { + if ns = strings.TrimSpace(ns); ns != "" { + args = append(args, "--dns", ns) + } + } + return args +} + // Stop releases every resource the harness created: the container, its // dedicated network, the per-instance fixture image tag (never a shared // base image), and the temp known_hosts directory. It is idempotent and diff --git a/internal/installer/harness/harness_test.go b/internal/installer/harness/harness_test.go index 8108022..6c57628 100644 --- a/internal/installer/harness/harness_test.go +++ b/internal/installer/harness/harness_test.go @@ -235,6 +235,21 @@ func TestSetDockerBin(t *testing.T) { SetDockerBin("") } +func TestDockerDNSOverride(t *testing.T) { + t.Setenv("RSMON_TEST_DOCKER_DNS", "") + if got := dockerDNS(); len(got) != 0 { + t.Fatalf("dockerDNS() with empty env = %v, want none", got) + } + t.Setenv("RSMON_TEST_DOCKER_DNS", "8.8.8.8, 1.1.1.1") + if got := dockerDNS(); len(got) != 4 || got[0] != "--dns" || got[1] != "8.8.8.8" || got[3] != "1.1.1.1" { + t.Fatalf("dockerDNS() = %v", got) + } + t.Setenv("RSMON_TEST_DOCKER_DNS", " ,,") + if got := dockerDNS(); len(got) != 0 { + t.Fatalf("dockerDNS() with blank entries = %v", got) + } +} + // writeStubDocker installs a fake docker binary that records its argv to // logPath and returns the recorded path. The stub succeeds for build, // network, run, and teardown calls; `port` fails so Start fails after diff --git a/internal/installer/harness/integration_test.go b/internal/installer/harness/integration_test.go index 06c7ef1..3ff8c5f 100644 --- a/internal/installer/harness/integration_test.go +++ b/internal/installer/harness/integration_test.go @@ -1,6 +1,7 @@ package harness import ( + "bytes" "context" "crypto/ed25519" "crypto/rand" @@ -14,6 +15,7 @@ import ( "golang.org/x/crypto/ssh" "golang.org/x/crypto/ssh/knownhosts" + "rocketgit.ru/rsmon/worker/internal/installer" "rocketgit.ru/rsmon/worker/internal/sshinstall" ) @@ -266,6 +268,236 @@ func TestHarnessHostKeyStable(t *testing.T) { } } +// TestSourceInstallFixtures is the work-package-3 acceptance test: each +// distro fixture starts clean (no Go, no worker source) and the real +// installer executes the full source flow over SSH - prerequisite +// install, verified Go toolchain download/extraction, clone/update of the +// public repository, resolved branch/commit record, and a build to a +// staging path. The running service and its config are deliberately not +// installed (that is work package 4). A rerun exercises idempotency. +// +// Opt-in: set RSMON_TEST_DOCKER=1 (see make test-ssh). +func TestSourceInstallFixtures(t *testing.T) { + SkipUnlessEnabled(t) + + for _, f := range Fixtures() { + f := f + t.Run(f.Name, func(t *testing.T) { + h, err := New("source-"+f.Name, f) + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute) + defer cancel() + if err := h.Start(ctx); err != nil { + t.Fatalf("start %s fixture: %v", f.Name, err) + } + t.Cleanup(func() { + if err := h.Stop(); err != nil { + t.Errorf("stop %s fixture: %v", f.Name, err) + } + }) + + // Clean state: no Go toolchain, no source, no leftover + // toolchain temp dirs. + client, err := h.Dial() + if err != nil { + t.Fatalf("dial %s fixture: %v", f.Name, err) + } + probe, err := RunCommand(client, "command -v go || true; test ! -e /usr/local/go && echo NO_GO; test ! -e /opt/rsmon-worker-src && echo NO_SOURCE; ls /tmp | grep -q rsmon-toolchain && echo LEAK; ls /usr/local 2>/dev/null | grep -qE 'go-staging|go-backup' && echo LEAK; echo DONE") + if err != nil { + t.Fatalf("clean-state probe: %v", err) + } + clean := string(probe) + if !strings.Contains(clean, "NO_GO") || !strings.Contains(clean, "NO_SOURCE") { + t.Fatalf("fixture is not clean: %q", clean) + } + if strings.Contains(clean, "LEAK") { + t.Fatalf("fixture has leftover toolchain temp dirs: %q", clean) + } + client.Close() //nolint:errcheck + + opts := installer.SourceInstallOptions{ + SSHOptions: installer.SSHOptions{ + Host: "127.0.0.1", + Port: h.Port(), + User: f.UserOrDefault(), + IdentityFile: testKeyPath(), + KnownHostsFile: h.KnownHostsPath(), + }, + } + if repo := strings.TrimSpace(os.Getenv("RSMON_TEST_SOURCE_REPO")); repo != "" { + opts.Repo = repo + } + if branch := strings.TrimSpace(os.Getenv("RSMON_TEST_SOURCE_BRANCH")); branch != "" { + opts.Branch = branch + } + + res, err := installer.SourceInstall(opts) + if err != nil { + t.Fatalf("source install on %s: %v", f.Name, err) + } + t.Logf("%s: %s -> branch=%s commit=%s staged=%s", f.Name, res.Detection.Summarize(), + res.ResolvedBranch, res.ResolvedCommit, res.StageBinary) + + if res.Detection.Distro != f.Distro || res.Detection.PackageManager != f.Pkg || res.Detection.InitSystem != f.Init { + t.Fatalf("detection = %+v, want %s/%s/%s", res.Detection, f.Distro, f.Pkg, f.Init) + } + if res.GoArch != "linux-"+strings.TrimPrefix(res.Plan.Toolchain.Arch, "linux-") { + t.Fatalf("resolved arch = %q, want %q", res.GoArch, res.Plan.Toolchain.Arch) + } + if res.ResolvedBranch == "" || len(res.ResolvedCommit) != 40 { + t.Fatalf("resolved branch/commit incomplete: %s @ %s", res.ResolvedBranch, res.ResolvedCommit) + } + if res.StageBinary == "" || res.RecordFile == "" || res.ToolchainDir == "" { + t.Fatalf("result paths incomplete: %+v", res) + } + + client, err = h.Dial() + if err != nil { + t.Fatalf("redial: %v", err) + } + defer client.Close() //nolint:errcheck + assertSourceInstallState(t, client, f.Name, res) + + // Idempotent rerun: succeeds, resolves the same branch, + // reuses the toolchain, and leaks no temp files. + res2, err := installer.SourceInstall(opts) + if err != nil { + t.Fatalf("source install rerun on %s: %v", f.Name, err) + } + if res2.ResolvedBranch != res.ResolvedBranch || len(res2.ResolvedCommit) != 40 { + t.Fatalf("rerun resolved = %s @ %s, want branch %s", res2.ResolvedBranch, res2.ResolvedCommit, res.ResolvedBranch) + } + assertSourceInstallState(t, client, f.Name, res2) + if out, err := RunCommand(client, "leak=0; ls /tmp | grep -q rsmon-toolchain && leak=1; ls /usr/local 2>/dev/null | grep -qE 'go-staging|go-backup' && leak=1; [ \"$leak\" -eq 1 ] && echo LEAK || echo CLEAN"); err != nil || strings.TrimSpace(string(out)) != "CLEAN" { + t.Fatalf("toolchain temp dirs leaked after rerun: %q, %v", out, err) + } + }) + } +} + +// assertSourceInstallState verifies the remote side-effects of a source +// install: the toolchain reports the pinned version, the staging binary +// exists and reports the resolved commit, and the record file carries +// the resolved branch and commit. +func assertSourceInstallState(t *testing.T, client *ssh.Client, name string, res *installer.SourceInstallResult) { + t.Helper() + + out, err := RunCommand(client, res.ToolchainDir+"/bin/go version") + if err != nil { + t.Fatalf("%s: toolchain missing: %v", name, err) + } + if !strings.Contains(string(out), "go"+res.Plan.Toolchain.Version) { + t.Fatalf("%s: toolchain version = %q, want go%s", name, out, res.Plan.Toolchain.Version) + } + + out, err = RunCommand(client, "test -x "+shellQuote(res.StageBinary)+" && echo BUILT") + if err != nil || !strings.Contains(string(out), "BUILT") { + t.Fatalf("%s: staging binary not present at %s: %q, %v", name, res.StageBinary, out, err) + } + out, err = RunCommand(client, shellQuote(res.StageBinary)+" --version") + if err != nil { + t.Fatalf("%s: staging binary --version: %v", name, err) + } + if !strings.Contains(string(out), "commit="+res.ResolvedCommit[:12]) { + t.Fatalf("%s: staging binary reports commit %q, want short %s", name, out, res.ResolvedCommit[:12]) + } + + record, err := RunCommand(client, "cat "+shellQuote(res.RecordFile)) + if err != nil { + t.Fatalf("%s: read record file: %v", name, err) + } + if !strings.Contains(string(record), "branch="+res.ResolvedBranch) || !strings.Contains(string(record), "commit="+res.ResolvedCommit) { + t.Fatalf("%s: record file = %q, want branch=%s commit=%s", name, record, res.ResolvedBranch, res.ResolvedCommit) + } +} + +// TestSourceInstallDirtyCheckoutPreservesStaging is the work-package-3 +// failure-atomicity test: after a successful install, dirtying the +// tracked working tree makes the next checkout fail closed. The rerun +// must report the checkout error without ever reaching the build step, +// leaving the previous staging binary and commit record byte-for-byte +// unchanged, and without leaking toolchain staging/backup directories. +func TestSourceInstallDirtyCheckoutPreservesStaging(t *testing.T) { + SkipUnlessEnabled(t) + + f := Fixtures()[0] // alpine is the smallest fixture + h, err := New("dirty-"+f.Name, f) + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute) + defer cancel() + if err := h.Start(ctx); err != nil { + t.Fatalf("start %s fixture: %v", f.Name, err) + } + t.Cleanup(func() { + if err := h.Stop(); err != nil { + t.Errorf("stop %s fixture: %v", f.Name, err) + } + }) + + opts := installer.SourceInstallOptions{ + SSHOptions: installer.SSHOptions{ + Host: "127.0.0.1", + Port: h.Port(), + User: f.UserOrDefault(), + IdentityFile: testKeyPath(), + KnownHostsFile: h.KnownHostsPath(), + }, + } + if repo := strings.TrimSpace(os.Getenv("RSMON_TEST_SOURCE_REPO")); repo != "" { + opts.Repo = repo + } + res, err := installer.SourceInstall(opts) + if err != nil { + t.Fatalf("initial source install: %v", err) + } + + client, err := h.Dial() + if err != nil { + t.Fatalf("dial: %v", err) + } + defer client.Close() //nolint:errcheck + + // Dirty a tracked file so the rerun's checkout refuses to proceed. + // Makefile differs between master and master~1 (unlike go.mod). + if _, err := RunCommand(client, "git -C "+shellQuote(res.Plan.BuildDir)+" checkout -q master~1 -- Makefile"); err != nil { + t.Fatalf("dirty the working tree: %v", err) + } + beforeBinary, err := RunCommand(client, "sha256sum "+shellQuote(res.StageBinary)) + if err != nil { + t.Fatal(err) + } + beforeRecord, err := RunCommand(client, "cat "+shellQuote(res.RecordFile)) + if err != nil { + t.Fatal(err) + } + + if _, err := installer.SourceInstall(opts); err == nil || !strings.Contains(err.Error(), "check out branch") { + t.Fatalf("rerun err = %v, want checkout failure on dirty tree", err) + } + + afterBinary, err := RunCommand(client, "sha256sum "+shellQuote(res.StageBinary)) + if err != nil { + t.Fatal(err) + } + afterRecord, err := RunCommand(client, "cat "+shellQuote(res.RecordFile)) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(bytes.TrimSpace(beforeBinary), bytes.TrimSpace(afterBinary)) { + t.Fatalf("staging binary changed after failed rerun:\nbefore: %s\nafter: %s", beforeBinary, afterBinary) + } + if !bytes.Equal(bytes.TrimSpace(beforeRecord), bytes.TrimSpace(afterRecord)) { + t.Fatalf("commit record changed after failed rerun:\nbefore: %s\nafter: %s", beforeRecord, afterRecord) + } + if out, err := RunCommand(client, "leak=0; ls /tmp | grep -q rsmon-toolchain && leak=1; ls /usr/local 2>/dev/null | grep -qE 'go-staging|go-backup' && leak=1; [ \"$leak\" -eq 1 ] && echo LEAK || echo CLEAN"); err != nil || strings.TrimSpace(string(out)) != "CLEAN" { + t.Fatalf("toolchain staging leaked after failed rerun: %q, %v", out, err) + } +} + // makeProber builds an sshinstall.FileProber over a live SSH session. func makeProber(client *ssh.Client) sshinstall.FileProber { return func(paths ...string) map[string]bool { diff --git a/internal/installer/sourceinstall.go b/internal/installer/sourceinstall.go new file mode 100644 index 0000000..3b867ac --- /dev/null +++ b/internal/installer/sourceinstall.go @@ -0,0 +1,558 @@ +package installer + +import ( + "errors" + "fmt" + "net" + "path/filepath" + "regexp" + "strconv" + "strings" + "time" + + "golang.org/x/crypto/ssh" + + "rocketgit.ru/rsmon/worker/internal/sshinstall" +) + +// SourceInstallOptions drives the remote source installation (work +// package 3 of docs/source-installation.md). It reuses the deploy +// command's SSHOptions for authentication and host-key verification and +// adds the source-build knobs. It never carries a worker token or +// control-plane credential: the built worker is staged, not configured +// or started, so no secret is ever sent to the remote host. +type SourceInstallOptions struct { + SSHOptions + + // Repo is the public worker repository to clone or update. Empty + // uses the sshinstall default. Only https URLs without userinfo are + // accepted. + Repo string + // Branch pins the branch to build. Empty resolves the remote's + // default branch (the public repo currently publishes "master"); + // the resolved branch and commit are recorded in the build dir. + Branch string + // GoVersion defaults to the pinned sshinstall toolchain (1.26.0). + // Non-default versions have no baked checksum yet and are rejected. + GoVersion string + // GoArch optionally pins the Go download archive suffix (e.g. + // "amd64"); empty derives it from the remote `uname -m`. + GoArch string + // BuildDir is the remote clone/build directory. + BuildDir string + // GoModuleProxy overrides GOPROXY for the remote build. + GoModuleProxy string + // ToolchainDir is where the verified Go toolchain is installed. + // It must be an absolute path ending in /go (default + // /usr/local/go). Replacement is atomic: the new toolchain is + // downloaded, verified, and staged before the prior one is moved + // aside, and the prior one is restored if the swap fails. + ToolchainDir string + // StageBinary is where the built worker binary is written. It must + // be absolute and defaults to /rsmon-worker. The running + // service and its config are NOT touched by this work package. + StageBinary string + // SessionTimeout bounds each remote command. 0 uses the default + // (30 minutes); the build step can legitimately run for minutes. + SessionTimeout time.Duration +} + +// SourceInstallResult is what a source installation resolved to. The +// resolved branch and commit are recorded on the remote host in +// RecordFile, and the built binary is left at StageBinary for the next +// (service-activation) work package to install atomically. +type SourceInstallResult struct { + Detection sshinstall.Detection + Plan sshinstall.SourcePlan + GoArch string // resolved archive suffix, e.g. "linux-amd64" + ToolchainDir string + ResolvedBranch string + ResolvedCommit string + RecordFile string + StageBinary string +} + +// defaultToolchainDir is the standard Go installation prefix. +const defaultToolchainDir = "/usr/local/go" + +// defaultSessionTimeout bounds each remote command when the operator +// does not configure one. The staging build and first-time module +// downloads can run for minutes, so this is generous. +const defaultSessionTimeout = 30 * time.Minute + +// commitRecordName is the file (inside BuildDir) that records the +// resolved branch and commit the build was produced from. +const commitRecordName = "rsmon-worker.commit" + +var ( + commitPattern = regexp.MustCompile(`^[0-9a-f]{40}$`) + branchNamePattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._/-]*$`) +) + +// sourceExecutor bundles the SSH client, privilege options, and per +// command timeout used by every remote source-install step. +type sourceExecutor struct { + client *ssh.Client + ssh SSHOptions + timeout time.Duration +} + +// runPrivileged executes a command through the SSHOptions privilege path +// (root, passwordless sudo, or sudo -S) and returns bounded stdout. +func (e *sourceExecutor) runPrivileged(command string) ([]byte, error) { + cmd, stdin := sudoWrap(e.ssh.User, e.ssh.SudoPassword, command) + return runRemoteOutput(e.client, cmd, stdin, e.timeout) +} + +// runPlain executes a command as the SSH user and returns bounded +// stdout. +func (e *sourceExecutor) runPlain(command string) ([]byte, error) { + return runRemoteOutput(e.client, command, nil, e.timeout) +} + +// fileProber builds the sshinstall.FileProber used for init-system +// detection over a live SSH session. +func (e *sourceExecutor) fileProber() sshinstall.FileProber { + return func(paths ...string) map[string]bool { + quoted := make([]string, len(paths)) + for i, p := range paths { + quoted[i] = shellQuote(p) + } + expr := "for p in " + strings.Join(quoted, " ") + "; do [ -e \"$p\" ] && printf '%s\\n' \"$p\"; done; true" + out, err := e.runPlain(expr) + if err != nil { + return nil + } + present := make(map[string]bool, len(paths)) + for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") { + if line = strings.TrimSpace(line); line != "" { + present[line] = true + } + } + return present + } +} + +// SourceInstall executes the source-install flow over SSH: it reuses the +// deploy command's SSH authentication and host-key verification, detects +// the remote host, plans the pinned toolchain and package prerequisites, +// then installs packages, downloads and verifies the Go toolchain, +// clones/updates the public repository (verifying an existing checkout's +// origin matches the configured repository), checks out the resolved +// branch, builds the worker to a staging path, and only then records the +// resolved branch and commit. +// +// The running service, its configuration, and its data directory are +// deliberately untouched: atomic activation and rollback are the next +// work package. Every remote step runs with the same privilege path as +// `deploy` (root, passwordless sudo, or sudo -S), every interpolated +// value is single-quoted, every step script fails closed (`set -eu` or +// explicit `&&`/retry), and every failure returns a bounded, actionable +// error. Each remote command is capped by SessionTimeout and its stdout +// is size-bounded. +func SourceInstall(opts SourceInstallOptions) (*SourceInstallResult, error) { + opts, err := normalizeSourceOptions(opts) + if err != nil { + return nil, err + } + auth, err := sshAuth(opts.SSHOptions) + if err != nil { + return nil, err + } + hostKey, err := hostKeyCallback(opts.SSHOptions) + if err != nil { + return nil, err + } + client, err := ssh.Dial("tcp", net.JoinHostPort(opts.Host, strconv.Itoa(opts.Port)), &ssh.ClientConfig{ + User: opts.User, + Auth: auth, + HostKeyCallback: hostKey, + Timeout: 15 * time.Second, + }) + if err != nil { + return nil, fmt.Errorf("connect to %s: %w", opts.Host, err) + } + defer client.Close() //nolint:errcheck + + executor := &sourceExecutor{client: client, ssh: opts.SSHOptions, timeout: opts.SessionTimeout} + + osRelease, err := executor.runPlain("cat /etc/os-release") + if err != nil { + return nil, fmt.Errorf("read remote /etc/os-release: %w", err) + } + uname, err := executor.runPlain("uname -m") + if err != nil { + return nil, fmt.Errorf("read remote machine architecture: %w", err) + } + detection := sshinstall.Detect(string(osRelease), executor.fileProber()) + plan, err := sshinstall.PlanSource(detection, sshinstall.SourceOptions{ + Repo: opts.Repo, + Branch: opts.Branch, + GoVersion: opts.GoVersion, + GoArch: opts.GoArch, + UnameM: strings.TrimSpace(string(uname)), + BuildDir: opts.BuildDir, + GoModuleProxy: opts.GoModuleProxy, + }) + if err != nil { + return nil, err + } + + stage := opts.StageBinary + if stage == "" { + stage = filepath.Join(plan.BuildDir, "rsmon-worker") + } + result := &SourceInstallResult{ + Detection: detection, + Plan: plan, + GoArch: plan.Toolchain.Arch, + ToolchainDir: opts.ToolchainDir, + StageBinary: stage, + } + + if len(plan.Packages) > 0 { + script := "sh -c " + shellQuote(packageScript(detection.PackageManager, plan.Packages)) + if _, err := executor.runPrivileged(script); err != nil { + return nil, fmt.Errorf("install prerequisites via %s: %w", detection.PackageManager, err) + } + } + + script := "sh -c " + shellQuote(toolchainScript(plan.Toolchain, opts.ToolchainDir)) + if _, err := executor.runPrivileged(script); err != nil { + return nil, fmt.Errorf("install Go %s toolchain: %w", plan.Toolchain.Version, err) + } + + // Clone or update. An existing checkout must point at the configured + // repository, or the install fails before fetching or building. + script = "sh -c " + shellQuote(cloneUpdateScript(plan.Repo, plan.BuildDir)) + if _, err := executor.runPrivileged(script); err != nil { + return nil, fmt.Errorf("clone/update source repository: %w", err) + } + + out, err := executor.runPrivileged("sh -c " + shellQuote(resolveBranchScript(plan.BuildDir))) + if err != nil { + return nil, fmt.Errorf("resolve remote default branch: %w", err) + } + branch, err := parseResolvedBranch(string(out)) + if err != nil { + return nil, err + } + if plan.Branch != "" && plan.Branch != branch { + ref := "refs/remotes/origin/" + plan.Branch + if _, err := executor.runPrivileged("sh -c " + shellQuote(refExistsScript(plan.BuildDir, ref))); err != nil { + return nil, fmt.Errorf("branch %q does not exist on the remote repository: %w", plan.Branch, err) + } + branch = plan.Branch + } + result.ResolvedBranch = branch + + // Checkout fails closed: a dirty tree or missing branch aborts before + // the build, so the previous staging binary is left untouched. + out, err = executor.runPrivileged("sh -c " + shellQuote(checkoutScript(plan.BuildDir, branch))) + if err != nil { + return nil, fmt.Errorf("check out branch %q: %w", branch, err) + } + commit, err := parseResolvedCommit(string(out)) + if err != nil { + return nil, fmt.Errorf("resolve commit: %w", err) + } + result.ResolvedCommit = commit + + // Build with the repository's own flags (same -ldflags shape the + // worker Makefile uses), resolving COMMIT from the checkout and + // BUILD_DATE at install time. The binary is built to a temp path, + // verified, and atomically swapped into the staging path; a failed + // build leaves the previous staging binary in place. + buildDate := time.Now().UTC().Format("2006-01-02T15:04:05Z") + ldflags := fmt.Sprintf("-s -w -X main.version=dev -X main.commit=%s -X main.buildDate=%s", commit[:12], buildDate) + script = "sh -c " + shellQuote(buildScript(opts.ToolchainDir, plan.BuildDir, plan.GoModuleProxy, stage, ldflags)) + if _, err := executor.runPrivileged(script); err != nil { + return nil, fmt.Errorf("build worker binary: %w", err) + } + + // The commit record is written only after a successful build, so the + // record and the staged binary always correspond to the same commit. + result.RecordFile = filepath.Join(plan.BuildDir, commitRecordName) + if _, err := executor.runPrivileged("sh -c " + shellQuote(commitRecordScript(plan.BuildDir, branch, commit))); err != nil { + return nil, fmt.Errorf("record resolved commit: %w", err) + } + return result, nil +} + +// normalizeSourceOptions validates operator input before any remote +// connection or mutation. Every value that later reaches a remote shell +// is constrained here. +func normalizeSourceOptions(o SourceInstallOptions) (SourceInstallOptions, error) { + if o.Host == "" || o.User == "" { + return o, errors.New("--host and --user are required for source install") + } + if o.Port == 0 { + o.Port = 22 + } + if o.Port < 1 || o.Port > 65535 { + return o, errors.New("SSH port must be between 1 and 65535") + } + if o.Branch != "" && !validBranchName(o.Branch) { + return o, fmt.Errorf("invalid branch %q: only A-Za-z0-9, dots, underscores, slashes, and hyphens are allowed", o.Branch) + } + if o.GoVersion != "" && !sshinstall.ValidGoVersion(o.GoVersion) { + return o, fmt.Errorf("invalid Go version %q: only digits, letters, dots, dashes, and underscores are allowed", o.GoVersion) + } + if o.GoArch != "" && !sshinstall.ValidGoArch(o.GoArch) { + return o, fmt.Errorf("invalid Go architecture %q: only letters, digits, dashes, and underscores are allowed", o.GoArch) + } + if err := sshinstall.ValidateRepoURL(o.Repo); err != nil { + return o, fmt.Errorf("invalid repository: %w", err) + } + if o.ToolchainDir == "" { + o.ToolchainDir = defaultToolchainDir + } + if !strings.HasPrefix(o.ToolchainDir, "/") || filepath.Base(o.ToolchainDir) != "go" { + return o, fmt.Errorf("toolchain directory must be an absolute path ending in /go, got %q", o.ToolchainDir) + } + if o.StageBinary != "" && !strings.HasPrefix(o.StageBinary, "/") { + return o, fmt.Errorf("staging binary path must be absolute, got %q", o.StageBinary) + } + if o.SessionTimeout <= 0 { + o.SessionTimeout = defaultSessionTimeout + } + return o, nil +} + +// validBranchName reports whether a branch is a safe git branch name +// that can be interpolated into remote commands. The charset check is +// the command-injection boundary; the extra rules reject git-invalid or +// ambiguous refname patterns. +func validBranchName(branch string) bool { + if !branchNamePattern.MatchString(branch) { + return false + } + if strings.HasPrefix(branch, "-") || strings.HasPrefix(branch, "/") || + strings.Contains(branch, "..") || strings.Contains(branch, "@{") || + strings.Contains(branch, "//") || strings.HasSuffix(branch, ".") || + strings.HasSuffix(branch, "/") { + return false + } + return true +} + +// parseResolvedBranch turns the `git symbolic-ref` output +// ("origin/master\n") into the short branch name, validating it so +// remote-controlled output can never inject a command. +func parseResolvedBranch(raw string) (string, error) { + branch := strings.TrimSpace(raw) + branch = strings.TrimPrefix(branch, "origin/") + if !validBranchName(branch) { + return "", fmt.Errorf("remote reported an invalid default branch %q", strings.TrimSpace(raw)) + } + return branch, nil +} + +// parseResolvedCommit extracts the 40-hex commit from `git rev-parse +// HEAD` output, taking the last whitespace-separated token so unrelated +// stdout cannot satisfy the parse. +func parseResolvedCommit(raw string) (string, error) { + fields := strings.Fields(strings.TrimSpace(raw)) + if len(fields) == 0 { + return "", errors.New("remote reported no commit") + } + commit := fields[len(fields)-1] + if !commitPattern.MatchString(commit) { + return "", fmt.Errorf("remote reported an invalid resolved commit %q", commit) + } + return commit, nil +} + +// quoteList renders each item as a single shell-quoted word. +func quoteList(items []string) string { + quoted := make([]string, len(items)) + for i, item := range items { + quoted[i] = shellQuote(item) + } + return strings.Join(quoted, " ") +} + +// packageScript installs the minimal build prerequisites with the +// distro's package manager. It is idempotent on every supported manager +// and never installs a C compiler (the worker builds with CGO disabled). +func packageScript(pkg sshinstall.PackageManager, pkgs []string) string { + quoted := quoteList(pkgs) + switch pkg { + case sshinstall.PkgApk: + return "apk add --no-cache " + quoted + case sshinstall.PkgApt: + // Ubuntu/Debian need a fresh package index before installing. + return "export DEBIAN_FRONTEND=noninteractive\napt-get update\napt-get install -y --no-install-recommends " + quoted + case sshinstall.PkgPacman: + return "pacman -Sy --noconfirm --needed " + quoted + case sshinstall.PkgDnf: + return "dnf install -y --setopt=install_weak_deps=False " + quoted + case sshinstall.PkgYum: + return "yum install -y " + quoted + default: + return "" + } +} + +// toolchainScript downloads the pinned Go toolchain, verifies its +// published SHA-256 before extraction, stages the extract on the same +// filesystem as the target, verifies the staged toolchain reports the +// target version, and only then swaps it into place. The prior toolchain +// (when present) is moved to a sibling backup first and is restored if +// the swap fails, so a failed download/verify/extract/swap always leaves +// the prior Go untouched. A present toolchain already reporting the +// target version is reused (idempotent rerun). Temp and staging +// directories are removed on success and failure. +func toolchainScript(tc sshinstall.Toolchain, toolchainDir string) string { + parent := filepath.Dir(toolchainDir) + want := "go" + tc.Version + var b strings.Builder + b.WriteString("set -eu\n") + b.WriteString("parent=" + shellQuote(parent) + "\n") + b.WriteString("toolchain=" + shellQuote(toolchainDir) + "\n") + b.WriteString("want=" + shellQuote(want) + "\n") + b.WriteString("if [ -x \"$toolchain/bin/go\" ]; then\n") + b.WriteString(" have=\"$($toolchain/bin/go version 2>/dev/null | awk '{print $3}')\"\n") + b.WriteString(" if [ \"$have\" = \"$want\" ]; then\n") + b.WriteString(" echo \"go toolchain already present: $have\"\n") + b.WriteString(" exit 0\n") + b.WriteString(" fi\n") + b.WriteString("fi\n") + b.WriteString("work=\"$(mktemp -d /tmp/rsmon-toolchain-XXXXXX)\"\n") + b.WriteString("staging=\"$(mktemp -d \"$parent/.go-staging-XXXXXX\")\"\n") + b.WriteString("trap 'rm -rf \"$work\" \"$staging\"' EXIT HUP INT TERM\n") + b.WriteString("archive=\"$work/go" + tc.Version + "." + tc.Arch + ".tar.gz\"\n") + b.WriteString("curl -fsSL --retry 3 --retry-delay 2 -o \"$archive\" " + shellQuote(tc.URL) + "\n") + b.WriteString("printf '%s %s\\n' " + shellQuote(tc.SHA256) + " \"$archive\" | sha256sum -c -\n") + b.WriteString("tar -C \"$staging\" -xzf \"$archive\"\n") + b.WriteString("staged=\"$($staging/go/bin/go version | awk '{print $3}')\"\n") + b.WriteString("if [ \"$staged\" != \"$want\" ]; then\n") + b.WriteString(" printf 'staged toolchain failed verification: %s\\n' \"$staged\" >&2\n") + b.WriteString(" exit 1\n") + b.WriteString("fi\n") + // Swap atomically on the same filesystem, preserving the prior + // toolchain in a sibling backup with rollback on failure. + b.WriteString("backup=\"\"\n") + b.WriteString("if [ -e \"$toolchain\" ]; then\n") + b.WriteString(" backup=\"$parent/.go-backup\"\n") + b.WriteString(" rm -rf \"$backup\"\n") + b.WriteString(" mv \"$toolchain\" \"$backup\"\n") + b.WriteString("fi\n") + b.WriteString("if ! mv \"$staging/go\" \"$toolchain\"; then\n") + b.WriteString(" if [ -n \"$backup\" ]; then\n") + b.WriteString(" mv \"$backup\" \"$toolchain\"\n") + b.WriteString(" fi\n") + b.WriteString(" exit 1\n") + b.WriteString("fi\n") + b.WriteString("if [ -n \"$backup\" ]; then\n") + b.WriteString(" rm -rf \"$backup\"\n") + b.WriteString("fi\n") + b.WriteString("\"$toolchain/bin/go\" version\n") + return b.String() +} + +// cloneUpdateScript clones the repository when missing and otherwise +// fetches the latest refs, so a rerun updates in place. Before fetching +// an existing checkout, it verifies the configured repository matches +// the checkout's `remote.origin.url` exactly, so the installer can never +// fetch or build an unconfigured repository. The clone/fetch is retried +// up to three times (2s apart) because real repositories can be +// transiently unreachable (DNS, TLS, or proxy hiccups). The script fails +// closed: any exhausted retry exits non-zero. +func cloneUpdateScript(repo, buildDir string) string { + return "set -u\n" + + "repo=" + shellQuote(repo) + "\n" + + "dir=" + shellQuote(buildDir) + "\n" + + "if [ ! -d \"$dir/.git\" ]; then\n" + + " attempt=0\n" + + " while [ \"$attempt\" -lt 3 ]; do\n" + + " if git clone \"$repo\" \"$dir\"; then\n" + + " exit 0\n" + + " fi\n" + + " attempt=$((attempt + 1))\n" + + " sleep 2\n" + + " done\n" + + " exit 1\n" + + "fi\n" + + "origin=\"$(git -C \"$dir\" config --get remote.origin.url || true)\"\n" + + "if [ \"$origin\" != \"$repo\" ]; then\n" + + " printf 'existing checkout origin does not match configured repository\\nconfigured: %s\\nfound: %s\\n' \"$repo\" \"$origin\" >&2\n" + + " exit 1\n" + + "fi\n" + + "attempt=0\n" + + "while [ \"$attempt\" -lt 3 ]; do\n" + + " if git -C \"$dir\" fetch --prune origin; then\n" + + " exit 0\n" + + " fi\n" + + " attempt=$((attempt + 1))\n" + + " sleep 2\n" + + "done\n" + + "exit 1\n" +} + +// resolveBranchScript prints the remote's default branch short name +// (with an "origin/" prefix) via origin/HEAD. It fails closed so a +// set-head failure aborts rather than resolving a stale default. +func resolveBranchScript(buildDir string) string { + return "set -eu\n" + + "git -C " + shellQuote(buildDir) + " remote set-head origin --auto >/dev/null\n" + + "git -C " + shellQuote(buildDir) + " symbolic-ref --short refs/remotes/origin/HEAD\n" +} + +// refExistsScript verifies a remote-tracking ref exists (exit 0) without +// emitting output. +func refExistsScript(buildDir, ref string) string { + return "git -C " + shellQuote(buildDir) + " show-ref --verify --quiet " + shellQuote(ref) +} + +// checkoutScript moves the local branch to the resolved remote branch +// and prints the resolved commit. It fails closed (`set -eu`): before the +// destructive `checkout -B` (which would silently discard local changes) +// it refuses when the tracked working tree is dirty, so a checkout +// failure can never be masked by a stale `rev-parse` from the previous +// checkout, and a failed checkout aborts before the build. +func checkoutScript(buildDir, branch string) string { + return "set -eu\n" + + "git -C " + shellQuote(buildDir) + " diff --quiet || { echo 'working tree has uncommitted changes; refusing to overwrite' >&2; exit 1; }\n" + + "git -C " + shellQuote(buildDir) + " diff --cached --quiet || { echo 'working tree has staged changes; refusing to overwrite' >&2; exit 1; }\n" + + "git -C " + shellQuote(buildDir) + " checkout -q -B " + shellQuote(branch) + " " + shellQuote("origin/"+branch) + "\n" + + "git -C " + shellQuote(buildDir) + " rev-parse HEAD\n" +} + +// commitRecordScript writes the resolved branch and commit to the build +// dir record file with a bounded, greppable format. It runs only after a +// successful build, so the record always matches the staged binary. +func commitRecordScript(buildDir, branch, commit string) string { + path := shellQuote(filepath.Join(buildDir, commitRecordName)) + format := shellQuote("branch=%s\\ncommit=%s\\n") + return "umask 022; printf " + format + " " + shellQuote(branch) + " " + shellQuote(commit) + + " > " + path + " && chmod 0644 " + path +} + +// buildScript builds the worker with the repository's own flags into the +// staging path and verifies the resulting binary runs. CGO is disabled, +// trimpath keeps the build reproducible, and both caches (GOCACHE and +// GOMODCACHE) live inside the build dir so reruns reuse them. The binary +// is built to a sibling temp path, verified, and only then atomically +// swapped over the previous staging binary, so a failed build never +// replaces it. +func buildScript(toolchainDir, buildDir, goproxy, stage, ldflags string) string { + var b strings.Builder + b.WriteString("set -eu\n") + b.WriteString("cd " + shellQuote(buildDir) + "\n") + b.WriteString("export PATH=" + shellQuote(toolchainDir+"/bin") + ":$PATH\n") + b.WriteString("export GOCACHE=" + shellQuote(buildDir+"/.gocache") + "\n") + b.WriteString("export GOMODCACHE=" + shellQuote(buildDir+"/.gomodcache") + "\n") + if goproxy != "" { + b.WriteString("export GOPROXY=" + shellQuote(goproxy) + "\n") + } + b.WriteString("stage=" + shellQuote(stage) + "\n") + b.WriteString("tmp=\"$stage.new\"\n") + b.WriteString("trap 'rm -f \"$tmp\"' EXIT HUP INT TERM\n") + b.WriteString("CGO_ENABLED=0 ") + b.WriteString(shellQuote(toolchainDir + "/bin/go")) + b.WriteString(" build -trimpath -ldflags=" + shellQuote(ldflags) + " -o \"$tmp\" ./cmd/rsmon-worker\n") + b.WriteString("\"$tmp\" --version\n") + b.WriteString("mv -f \"$tmp\" \"$stage\"\n") + return b.String() +} diff --git a/internal/installer/sourceinstall_ssh_test.go b/internal/installer/sourceinstall_ssh_test.go new file mode 100644 index 0000000..a505621 --- /dev/null +++ b/internal/installer/sourceinstall_ssh_test.go @@ -0,0 +1,351 @@ +package installer + +import ( + "crypto/ed25519" + "crypto/rand" + "fmt" + "net" + "strings" + "sync" + "testing" + "time" + + "golang.org/x/crypto/ssh" + + "rocketgit.ru/rsmon/worker/internal/sshinstall" +) + +const ( + fakeSSHPassword = "fake-ssh-password" + fakeCommitHex = "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef" + fakeOsRelease = "ID=ubuntu\nNAME=\"Ubuntu\"\nVERSION_ID=24.04\n" +) + +// fakeSSHServer is a minimal in-process SSH server that simulates a +// remote Linux host for the SourceInstall orchestration tests. It uses +// real golang.org/x/crypto/ssh transport (no mocked SSH library), so the +// executor's dial, session, exec, and stdout/stderr plumbing is +// exercised end to end, and it records every command it ran. +type fakeSSHServer struct { + addr string + onExec func(command string) (stdout, stderr string, code int) + mu sync.Mutex + commands []string +} + +func startFakeSSHServer(t *testing.T, onExec func(command string) (string, string, int)) *fakeSSHServer { + t.Helper() + _, priv, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + t.Fatal(err) + } + signer, err := ssh.NewSignerFromKey(priv) + if err != nil { + t.Fatal(err) + } + config := &ssh.ServerConfig{ + PasswordCallback: func(_ ssh.ConnMetadata, pass []byte) (*ssh.Permissions, error) { + if string(pass) == fakeSSHPassword { + return nil, nil + } + return nil, fmt.Errorf("password rejected") + }, + } + config.AddHostKey(signer) + + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = ln.Close() }) + + srv := &fakeSSHServer{addr: ln.Addr().String(), onExec: onExec} + go func() { + for { + conn, err := ln.Accept() + if err != nil { + return + } + go srv.handleConn(conn, config) + } + }() + return srv +} + +func (s *fakeSSHServer) Port() int { + _, port, err := net.SplitHostPort(s.addr) + if err != nil { + return 0 + } + p := 0 + fmt.Sscanf(port, "%d", &p) + return p +} + +func (s *fakeSSHServer) Commands() []string { + s.mu.Lock() + defer s.mu.Unlock() + return append([]string(nil), s.commands...) +} + +func (s *fakeSSHServer) handleConn(conn net.Conn, config *ssh.ServerConfig) { + sconn, chans, reqs, err := ssh.NewServerConn(conn, config) + if err != nil { + return + } + defer sconn.Close() //nolint:errcheck + go ssh.DiscardRequests(reqs) + for newChannel := range chans { + if newChannel.ChannelType() != "session" { + _ = newChannel.Reject(ssh.UnknownChannelType, "unknown channel type") + continue + } + channel, requests, err := newChannel.Accept() + if err != nil { + continue + } + go func() { + defer channel.Close() + s.handleSession(channel, requests) + }() + } +} + +func (s *fakeSSHServer) handleSession(channel ssh.Channel, requests <-chan *ssh.Request) { + for req := range requests { + if req.Type != "exec" { + if req.WantReply { + _ = req.Reply(false, nil) + } + continue + } + var payload struct{ Command string } + if err := ssh.Unmarshal(req.Payload, &payload); err != nil { + if req.WantReply { + _ = req.Reply(false, nil) + } + continue + } + if req.WantReply { + _ = req.Reply(true, nil) + } + s.mu.Lock() + s.commands = append(s.commands, payload.Command) + s.mu.Unlock() + s.execCommand(channel, payload.Command) + return + } +} + +func (s *fakeSSHServer) execCommand(channel ssh.Channel, command string) { + handler := s.onExec + if handler == nil { + handler = defaultFakeExec + } + stdout, stderr, code := handler(command) + _, _ = channel.Write([]byte(stdout)) + _, _ = channel.Stderr().Write([]byte(stderr)) + _, _ = channel.SendRequest("exit-status", false, ssh.Marshal(struct{ Status uint32 }{uint32(code)})) + _ = channel.CloseWrite() +} + +// defaultFakeExec simulates a minimal Linux host: it answers os-release, +// uname, the init-marker file probe, origin URL, branch resolution, and +// rev-parse, and accepts every install step. A pinned-branch existence +// check (show-ref) fails by default so the missing-branch path is +// exercised without extra setup. +func defaultFakeExec(command string) (string, string, int) { + switch { + case strings.Contains(command, "cat /etc/os-release"): + return fakeOsRelease, "", 0 + case strings.Contains(command, "uname -m"): + return "x86_64\n", "", 0 + case strings.Contains(command, "[ -e"): + return "/usr/lib/systemd/system\n", "", 0 + case strings.Contains(command, "remote.origin.url"): + return sshinstall.DefaultRepo + "\n", "", 0 + case strings.Contains(command, "symbolic-ref"): + return "origin/master\n", "", 0 + case strings.Contains(command, "rev-parse HEAD"): + return fakeCommitHex + "\n", "", 0 + case strings.Contains(command, "show-ref"): + return "", "branch not found", 1 + default: + return "", "", 0 + } +} + +func testSSHOptions(port int) SourceInstallOptions { + return SourceInstallOptions{ + SSHOptions: SSHOptions{ + Host: "127.0.0.1", + Port: port, + User: "root", + Password: fakeSSHPassword, + InsecureHostKey: true, + }, + } +} + +func TestSourceInstallSSHFlow(t *testing.T) { + srv := startFakeSSHServer(t, nil) + res, err := SourceInstall(testSSHOptions(srv.Port())) + if err != nil { + t.Fatal(err) + } + + if res.Detection.Distro != sshinstall.DistroUbuntu || res.Detection.PackageManager != sshinstall.PkgApt { + t.Fatalf("detection = %+v", res.Detection) + } + if res.Detection.InitSystem != sshinstall.InitSystemd { + t.Fatalf("init detection = %q, want systemd", res.Detection.InitSystem) + } + if res.Plan.Toolchain.Arch != "linux-amd64" || res.Plan.Toolchain.Version != "1.26.0" { + t.Fatalf("toolchain = %+v", res.Plan.Toolchain) + } + if len(res.Plan.Packages) == 0 || res.Plan.Repo == "" { + t.Fatalf("plan = %+v", res.Plan) + } + if res.ResolvedBranch != "master" || res.ResolvedCommit != fakeCommitHex { + t.Fatalf("resolved = %s @ %s", res.ResolvedBranch, res.ResolvedCommit) + } + if res.ToolchainDir != "/usr/local/go" || res.StageBinary != "/opt/rsmon-worker-src/rsmon-worker" || + res.RecordFile != "/opt/rsmon-worker-src/rsmon-worker.commit" { + t.Fatalf("paths = %+v", res) + } + + commands := srv.Commands() + var joined strings.Builder + for _, c := range commands { + joined.WriteString(c) + joined.WriteString("\n") + } + // Marker substrings that survive the nested `sh -c '