diff --git a/README.md b/README.md index 5b36b38..6d531c2 100644 --- a/README.md +++ b/README.md @@ -183,11 +183,18 @@ A Go SSH source installer is implemented as the `source-install` subcommand: it detects the remote distro and architecture over the existing SSH transport, installs the minimal build prerequisites, downloads and SHA-256-verifies the pinned Go 1.26 toolchain, clones/updates the public repository, records the -resolved branch and commit, and builds the worker to a staging path. It -deliberately stops before touching the running service or its configuration -(atomic activation and rollback are the next milestone). The pure -detection/planning layer and the Docker/OpenSSH test harness that accepts it -are implemented; see +resolved branch and commit, builds the worker to a staging path, and then +atomically activates the staged binary together with the validated environment, +data directory, and the detected init's service definition. It starts the +worker and verifies the process and `/healthz`; any activation, start, or +health failure rolls back to the previous working install (preserving the +prior binary/env/unit and its enable state), concurrent activations are +serialized by a lock, a run killed mid-flight is recovered from its +leftover backup marker on the next activation, and reruns are idempotent +(exactly one worker process, no leaked temp files). Pass `--no-activate` +to stop at the staging build, and `--no-start` to install the layout +without starting the worker. The pure detection/planning layer and the +Docker/OpenSSH test harness that accepts it are implemented; see [`docs/source-installation.md`](docs/source-installation.md); run the live fixture matrix with `make test-ssh`. @@ -195,7 +202,8 @@ fixture matrix with `make test-ssh`. ./bin/rsmon-worker source-install \ --host worker.example.com \ --user deploy \ - --identity-file ~/.ssh/id_ed25519 + --identity-file ~/.ssh/id_ed25519 \ + --token-file ./worker.token ``` By default the installer builds the remote's default branch and records what it @@ -204,13 +212,21 @@ resolves to (the public repository currently publishes `master`). Pass install fails before building. The repository must be an `https://` URL without userinfo. -The built binary is left at `/opt/rsmon-worker-src/rsmon-worker` (override with +The built binary is staged at `/opt/rsmon-worker-src/rsmon-worker` (override with `--build-dir` / `--stage-binary`), the toolchain at `/usr/local/go` (replaced atomically: download, verify, stage, swap with rollback), and the resolved branch and commit in `/opt/rsmon-worker-src/rsmon-worker.commit` (written only -after a successful build). The same SSH auth, secret-file, and host-key options -as `deploy` apply; no worker token is sent because the source install does not -configure a service. Prefer `--key-passphrase-file`, `--password-file`, and +after a successful build). Activation installs the binary to +`/usr/local/bin/rsmon-worker`, the environment to +`/etc/rsmon-worker/worker.env` (mode 0600), the data directory to +`/var/lib/rsmon-worker`, and a systemd unit or OpenRC init script for the +detected init (an embedded supervisor manages the process when no init is +running, as in containers). The env and unit are uploaded into a server-side +0700 `mktemp -d` directory so no local user can race a predictable `/tmp` path, +and the env file is read/rendered exactly once. The same SSH auth, +secret-file, and host-key options as `deploy` apply; the worker token is +supplied with `--token`/`--token-file` and written only to the mode-0600 env +file. Prefer `--key-passphrase-file`, `--password-file`, and `--sudo-password-file` over their direct-flag equivalents: file options keep secrets out of the process list and shell history. diff --git a/cmd/rsmon-worker/management.go b/cmd/rsmon-worker/management.go index 870c6d5..07735ab 100644 --- a/cmd/rsmon-worker/management.go +++ b/cmd/rsmon-worker/management.go @@ -156,16 +156,21 @@ func secretValue(direct, path string) (string, error) { return strings.TrimRight(string(b), "\r\n"), nil } -// sourceInstallCommand drives the remote source installer (work package -// 3 of docs/source-installation.md): prerequisites, verified Go -// toolchain, clone/update, resolved branch/commit record, and a staging -// build. It reuses the deploy SSH options; it never touches the running -// service, configuration, or data directory on the remote host. +// sourceInstallCommand drives the remote source installer +// (docs/source-installation.md): prerequisites, verified Go toolchain, +// clone/update, resolved branch/commit record, a staging build, and (work +// package 4) the atomic activation of the staged binary, validated +// environment, data dir, and the detected init's service definition, +// followed by a start and a process + /healthz verification. It reuses +// the deploy SSH options. Any activation/start/health failure rolls back +// to the prior working install; --no-activate keeps the staging-only +// behavior. func sourceInstallCommand(args []string) int { fs := flag.NewFlagSet("source-install", flag.ContinueOnError) fs.SetOutput(os.Stderr) var opts installer.SourceInstallOptions - var passphraseFile, passwordFile, sudoPasswordFile string + var passphraseFile, passwordFile, sudoPasswordFile, tokenFile, workerPasswordFile string + var noActivate bool fs.StringVar(&opts.Host, "host", "", "SSH server hostname or address") fs.IntVar(&opts.Port, "port", 22, "SSH server port") fs.StringVar(&opts.User, "user", "", "SSH username") @@ -188,8 +193,22 @@ func sourceInstallCommand(args []string) int { fs.StringVar(&opts.ToolchainDir, "toolchain-dir", "", "remote Go install path ending in /go (default /usr/local/go)") fs.StringVar(&opts.StageBinary, "stage-binary", "", "staging binary path (default /rsmon-worker)") fs.DurationVar(&opts.SessionTimeout, "session-timeout", 0, "per-remote-command timeout (default 30m; 0 uses the default)") + fs.BoolVar(&noActivate, "no-activate", false, "stop after the staging build and do not install/start a service (activation is the default)") + fs.StringVar(&opts.Activation.Name, "name", "", "instance name: activates rsmon-worker- with its own config/data/unit/port") + fs.StringVar(&opts.Activation.URL, "url", installer.DefaultURL, "RSMon server URL (RSMON_URL)") + fs.StringVar(&opts.Activation.Token, "token", "", "worker API token (RSMON_TOKEN; required for activation)") + fs.StringVar(&opts.Activation.Token, "api-key", "", "worker API token (alias for --token)") + fs.StringVar(&tokenFile, "token-file", "", "file containing the worker API token") + fs.StringVar(&opts.Activation.PublicURL, "public-url", "", "advertised public origin (PUBLIC_URL; scheme + host, no path)") + fs.StringVar(&opts.Activation.Host, "worker-host", "", "worker webapp bind address (WORKER_HOST; default 127.0.0.1)") + fs.StringVar(&opts.Activation.Port, "worker-port", "", "worker webapp bind port (WORKER_PORT; default 27401, required with --name)") + fs.StringVar(&opts.Activation.Login, "worker-login", "", "operator console login (WORKER_LOGIN; default admin with a generated password)") + fs.StringVar(&opts.Activation.Password, "worker-password", "", "operator console password (WORKER_PASSWORD)") + fs.StringVar(&workerPasswordFile, "worker-password-file", "", "file containing the operator console password") + fs.StringVar(&opts.Activation.EnvFile, "env-file", "", "local systemd-safe env file uploaded for activation (overrides the individual knobs)") + fs.BoolVar(&opts.Activation.NoStart, "no-start", false, "install the binary, env, data dir, and service definition without starting the worker") fs.Usage = func() { - fmt.Fprintln(fs.Output(), "Usage: rsmon-worker source-install --host HOST --user USER [SSH options] [source options]") + fmt.Fprintln(fs.Output(), "Usage: rsmon-worker source-install --host HOST --user USER [--token TOKEN|--token-file FILE] [SSH options] [source options]") fs.PrintDefaults() } if err := fs.Parse(args); err != nil { @@ -215,6 +234,15 @@ func sourceInstallCommand(args []string) int { fmt.Fprintf(os.Stderr, "source-install failed: %v\n", err) return 1 } + if opts.Activation.Token, err = secretValue(opts.Activation.Token, tokenFile); err != nil { + fmt.Fprintf(os.Stderr, "source-install failed: %v\n", err) + return 1 + } + if opts.Activation.Password, err = secretValue(opts.Activation.Password, workerPasswordFile); err != nil { + fmt.Fprintf(os.Stderr, "source-install failed: %v\n", err) + return 1 + } + opts.Activation.Activate = !noActivate res, err := installer.SourceInstall(opts) if err != nil { fmt.Fprintf(os.Stderr, "source-install failed: %v\n", err) @@ -227,6 +255,22 @@ func sourceInstallCommand(args []string) int { fmt.Printf(" commit: %s\n", res.ResolvedCommit) fmt.Printf(" record file: %s\n", res.RecordFile) fmt.Printf(" staged build: %s\n", res.StageBinary) - fmt.Println(" status: not installed as a service (activation is the next milestone)") + if res.Activation != nil { + fmt.Printf(" installed to: %s\n", res.Activation.Binary) + fmt.Printf(" env file: %s (mode 0600)\n", res.Activation.EnvFile) + fmt.Printf(" data dir: %s\n", res.Activation.DataDir) + if res.Activation.UnitFile != "" { + fmt.Printf(" service: %s (%s)\n", res.Activation.UnitFile, res.Activation.UnitName) + } else { + fmt.Println(" service: none (no supported init system detected; supervisor-managed)") + } + if res.Activation.Started { + fmt.Printf(" status: running (supervisor=%s, /healthz verified)\n", res.Activation.Supervisor) + } else { + fmt.Printf(" status: installed (not started, --no-start)\n") + } + } else { + fmt.Println(" status: not installed as a service (--no-activate)") + } return 0 } diff --git a/docs/source-installation.md b/docs/source-installation.md index 2e48f64..228959e 100644 --- a/docs/source-installation.md +++ b/docs/source-installation.md @@ -2,10 +2,7 @@ ## Status -In progress. Work package 1 (reusable Docker/OpenSSH harness and distro -fixtures), the pure detection/planning foundation (work package 2 core), -and work package 3 (remote execution through the existing SSH transport) -are implemented: +In progress. Work packages 1-4 are implemented: - `internal/installer/harness` builds and runs real OpenSSH containers for Alpine, Ubuntu, and Arch, waits for real network readiness, captures @@ -24,17 +21,21 @@ are implemented: downloads and SHA-256-verifies the pinned Go toolchain before extraction, clones/updates the public repository, checks out the resolved branch, records the resolved branch and commit, and builds the - worker to a staging path. It deliberately does not install or replace - the running service, configuration, or data directory: atomic - activation and rollback are the next work package. + worker to a staging path. +- Work package 4 (`installer.SourceInstall` with `Activation` enabled) + atomically installs the staged binary, validated environment, data + directory, and the detected init's service definition; starts/restarts + the worker; verifies the process and `/healthz`; and rolls back to the + prior working install on any activation/start/health failure. Reruns + are idempotent with exactly one running worker and no leaked temp + files. The CLI runs the full flow by default and accepts + `--no-activate` (staging only) and `--no-start`. -The current Go installer can still only upload a binary or deploy an -immutable Docker image over SSH. Source installs now build remotely to a -staging path, but service activation over SSH (work package 4) is not -implemented yet; the acceptance test stops after install-to-staging and -idempotent-rerun assertions. Existing tests are unit tests plus the -harness tests that run against live OpenSSH containers when explicitly -enabled. +The current Go installer can also upload a binary or deploy an immutable +Docker image over SSH (`deploy`). Source installs build remotely, then +activate the staged build atomically with rollback. Existing tests are +unit tests plus the harness tests that run against live OpenSSH +containers when explicitly enabled. ## Initial Platform Scope @@ -66,7 +67,6 @@ It then: 7. atomically installs the binary, validated environment, data directory, and service definition; 8. starts the service and verifies process status and `/healthz`. - Repository, branch, Go version, checksum source, build directory, and Go module proxy may be configurable, but production output records their resolved values. The default repository is publicly readable and requires no source credential. @@ -79,7 +79,7 @@ Work package 3 is `installer.SourceInstall` in verification) and runs every remote step with the same privilege path (root, passwordless sudo, or `sudo -S -p ''` with the password delivered only over stdin). Steps 1-6 of the flow above are implemented; step 7 -(atomic install) is deliberately the next work package. +(atomic install) and step 8 (start + verify) are work package 4. Per step: @@ -162,8 +162,89 @@ Security properties of work package 3: or `/usr/local/.go-staging-*`/`.go-backup` leaks after the rerun. - Failed builds and failed checkouts leave the previous staging binary untouched (the binary is only overwritten by an atomically-swapped - successful build, and a checkout failure aborts before the build); - there is no running service or configuration to preserve yet. + successful build, and a checkout failure aborts before the build). + +## Work Package 4: Atomic Activation And Rollback + +Work package 4 is the second half of `installer.SourceInstall`, driven by +`Activation` (`internal/installer/sourceactivate.go`). It runs over the +same SSH executor, privilege path, and bounded-error machinery as the +staging steps and installs the staged build atomically: + +- **Layout.** The classic installer's on-disk layout is reused + (`resolvePaths`): binary `/usr/local/bin/rsmon-worker[-name]`, config + `/etc/rsmon-worker[-name]/worker.env` (mode 0600), data + `/var/lib/rsmon-worker[-name]`, and a service definition for the + detected init (systemd unit `/etc/systemd/system/rsmon-worker.service` + mode 0644, or an OpenRC script `/etc/init.d/rsmon-worker` mode 0755). + A named instance (`--name`) mirrors the classic multi-instance + convention with `-` suffixes and its own unit and port. +- **Environment.** The env file is rendered by the classic installer's + `resolveInstallEnv` + `renderEnvFile` (so `PUBLIC_URL` canonicalization, + basic-auth XOR, required `RSMON_URL`/`RSMON_TOKEN`, and systemd-safe + value validation are identical to `install`). The env file is read + *exactly once* and validated from the in-memory bytes + (`parseEnvironmentContent`), and the rendered env is cached during + option normalization and reused at activation time, so a hostile local + writer cannot swap the file between the preflight and the remote + install (env-file TOCTOU). The rendered bytes are uploaded into a + server-created 0700 temp dir (`mktemp -d /tmp/rsmon-worker-act.XXXXXX` + under the SSH user) rather than a predictable `/tmp` path, eliminating + the symlink/TOCTOU attack surface on the upload while the token still + travels only as base64 over the session stdin and, later, only inside + the mode-0600 env file. The env is then installed with mode 0600 in the + config dir (mode 0750). +- **Atomic install.** The staged binary is validated (` + --version`) before any state is touched, then installed with a + same-directory temp file + rename. The env and unit files are installed + the same way. The data dir (and `webapp/` subdir) is created with mode + 0755. +- **Supervisor.** `restart_svc` drives start/restart through whichever + init system is *actually running*: systemd (`systemctl`), OpenRC + (`rc-service`), or the embedded supervisor fallback when no init is + running (containers/chroots, the "no-service gate"). The health loop + verifies the service is active *through the supervisor* — `systemctl + is-active` for systemd, `rc-service status` for OpenRC, and a + zombie-aware pid check for the embedded supervisor — never through a + pid file on the init-managed paths. The embedded supervisor keeps the + worker as a single background process with a pid file under the data + dir, stops the previous instance before starting, refuses to kill a pid + whose executable is not the configured worker (`/proc//exe` + checked before every `kill`), and answers `/healthz` via the worker's + own `liveness` subcommand with a clean `env -i` environment built from + the env file (so a rolled-back env can never leak a stale variable into + the restored process). The OpenRC init script exports the env-file + variables (`set -a` before sourcing) so the worker inherits them when + OpenRC starts it. +- **Rollback.** The prior binary, env, and unit (and the unit's enable + state) are snapshotted into the data dir before any mutation, and an + `EXIT`/`HUP`/`INT`/`TERM` trap restores them (preserving the prior + binary/env/unit metadata via `cp -p`) and restarts the prior service if + any later step fails or a signal interrupts the run. A corrupt staged + binary fails the pre-validation *before* the trap is armed, so the + prior install is never touched. A fresh-install failure disables the + newly-enabled unit (`systemctl disable` / `rc-update del`) and removes + it; a rerun failure restores the prior unit and re-applies its prior + enable state. Rollback also removes the backup dir, the `.new` temp + files, the uploaded env/unit temps, and the activation lock. +- **Lock, recovery, interruption.** A `mkdir`-based activation lock + (`.rsmon-activate.lock`, broken automatically when its recorded pid is + dead) prevents concurrent activations from racing the deployed-state + mutation. The snapshot is marked; a run killed mid-flight (SSH drop, + SIGKILL) leaves that marker, and the next activation restores the + leftover snapshot before proceeding, so the host never stays + half-activated. +- **Idempotency.** A rerun reuses the existing env (rewriting it + deterministically from the same knobs), swaps the binary atomically, + restarts exactly one worker, and leaves no `.rsmon-backup`, `.new`, + `/tmp/rsmon-worker-act.*`, or lock leftovers. `--no-start` installs the + full layout without starting anything and reruns stay idle. + +The `source-install` CLI activates by default; pass `--no-activate` for +the staging-only behavior of work package 3. The worker token is required +for activation and is supplied with `--token`/`--token-file` or +`--env-file` (prefer the file options; direct flags expose the value +through the process list). ## Docker OpenSSH Test Harness @@ -273,22 +354,37 @@ RSMON_TEST_DOCKER=1 go test -v -count=1 -timeout 30m ./internal/installer/harnes matches, the clone's origin is verified against the configured repository and then fetched in place, the resolved branch/commit record is rewritten after the new build succeeds, and the staging build atomically swaps over the - previous staging binary. The "one active service" guarantee is a work package - 4 property (activation); work package 3 leaves no running service to - duplicate and no leaked toolchain temp files. + previous staging binary. Activation (work package 4) makes the rerun + idempotent at the service level too: exactly one worker process exists, the + env file is rewritten deterministically from the same knobs, and no + `.rsmon-backup`, `.new`, `/tmp/rsmon-worker-act.*`, or lock files leak. - Wrong host fingerprints fail before remote mutation. The harness's fresh known_hosts file is trust-on-first-use; its dedicated mismatch test dials against a known_hosts entry carrying a different server key and proves the dial fails before any command runs. - Tokens/passwords should come from files: `--key-passphrase-file`, - `--password-file`, and `--sudo-password-file` keep secrets out of argv and - shell history, while the equivalent direct flags expose them through the - process list. The source install itself sends no worker token or - control-plane credential at all, and sudo passwords travel only over the - session's stdin. -- Remote temporary files are removed on success and failure. + `--password-file`, `--sudo-password-file`, and the activation + `--token-file`/`--env-file` keep secrets out of argv and shell history, + while the equivalent direct flags expose them through the process list. The + worker token is written only to the mode-0600 env file, never echoed into a + command or log, and sudo passwords travel only over the session's stdin. The + env and unit uploads land in a server-created 0700 `mktemp -d` directory (not + a predictable `/tmp` path), so no local user can plant a symlink at the + upload target (TOCTOU), and the env file itself is read/rendered exactly once + so a local writer cannot swap it between validation and upload. +- Remote temporary files are removed on success and failure, including the + activation backup dir, the uploaded env/unit temps, and the activation lock. + The snapshot is written under a marker so a run killed mid-flight is + recovered (restored) by the next activation instead of leaving the host + half-activated. - Failed builds, failed checkouts, and failed toolchain swaps do not replace a - working binary or service definition. + working binary or service definition. Failed activations, failed starts, and + failed `/healthz` verifications restore the prior binary, env, and service + definition (preserving their metadata), re-apply the prior unit enable state + (or disable a freshly-enabled unit on a fresh failure), and bring the prior + worker back up (work package 4 rollback). The supervisor only ever kills a + process whose `/proc//exe` matches the configured worker binary, so a + stale or recycled pid file can never kill an unrelated process. - Package-manager and download failures return bounded actionable errors. - The installer verifies Go tarball checksum before extraction. @@ -300,24 +396,32 @@ RSMON_TEST_DOCKER=1 go test -v -count=1 -timeout 30m ./internal/installer/harnes - [x] 3. Execute source installation through the existing SSH transport (prerequisites, verified Go toolchain, clone/update, resolved branch and commit record, and a build to a staging path; no service activation). -- [ ] 4. Add atomic build/install, idempotency, and failure rollback. +- [x] 4. Add atomic build/install, idempotency, and failure rollback (binary, + env, data dir, and init service definition installed atomically; process + and `/healthz` verified; activation/start/health failures restore the + prior install; reruns keep exactly one service). - [ ] 5. Add Alpine, Ubuntu, and Arch network E2E tests to CI. - [ ] 6. Add CentOS-family support. - [ ] 7. Plan native Windows service and macOS launchd installers separately. ## Acceptance Gates -- [ ] All three initial Linux images install from a clean state through OpenSSH. - Work package 3 covers the install-to-staging half (package install, - verified Go 1.26, clone/update, resolved commit, staging build); the - service-start half is the work package 4 gate. -- [ ] The built worker reports the expected version/commit and serves `/healthz`. -- [ ] Re-running the installer succeeds without duplicate services or leaked files. -- [ ] Host-key, checksum, clone, build, and service-start failure tests preserve the - previous installation. +- [x] All three initial Linux images install from a clean state through OpenSSH: + package install, verified Go 1.26, clone/update, resolved commit, staging + build, and (work package 4) atomic activation with a running verified + worker. +- [x] The built worker reports the expected version/commit and serves `/healthz` + (verified by the E2E fixture tests and by the activation health gate). +- [x] Re-running the installer succeeds without duplicate services or leaked + files (exactly one worker process and no backup/temp leftovers asserted by + the rerun test on all three fixtures). +- [x] Host-key, checksum, clone, build, and service-start failure tests preserve + the previous installation (build/checkout, activation, start, and health + failure reruns on the Alpine fixture all restore the prior binary, env, + and running service). - [ ] CI uses approved registry mirrors and cleans every test container/network. -## Verified Test Evidence (work packages 1-3) +## Verified Test Evidence (work packages 1-4) Recorded 2026-08-12 from `make test-ssh` (Docker Engine 29.7.1): @@ -334,14 +438,48 @@ Recorded 2026-08-12 from `make test-ssh` (Docker Engine 29.7.1): prerequisite install, verified Go 1.26 toolchain download/extraction, clone of the public repository, resolution of its default branch, resolved-commit record file, and a staging build that reports the resolved commit via - `--version`. The public repo's `HEAD` was `master` at `4651deb2...` during - the run, and all three fixtures resolved to that branch and commit (the - installer records whatever the remote publishes). A rerun succeeds, keeps - the same branch, reuses the toolchain, and leaves no - `/tmp/rsmon-toolchain-*` temp dirs. The running service and its - configuration are not touched (work package 4 boundary). -- The unit suite covers the per-step remote scripts, option validation, - branch/commit parsing, sudo wrapping (password never in the command), the - secrets-absent contract, and an in-process real-SSH orchestration flow with - failure paths for missing pinned branches, build failures, and detection - failures. + `--version`. A rerun succeeds, keeps the same branch, reuses the toolchain, + and leaves no `/tmp/rsmon-toolchain-*` temp dirs. +- `TestSourceInstallActivationFixtures` runs the full work-package-4 flow on + each fixture from a clean state: after the staging build the installer + atomically installs the binary (`/usr/local/bin/rsmon-worker`), env + (`/etc/rsmon-worker/worker.env`, mode 0600; config dir 0750), data dir + (`/var/lib/rsmon-worker`), and the detected init's service definition + (systemd unit on Ubuntu/Arch, OpenRC script on Alpine), then starts the + worker via the embedded supervisor (no init runs inside the fixtures) and + verifies exactly one worker process and a live `/healthz`. A rerun succeeds, + keeps a single process, rewrites the env deterministically, and leaves no + `.rsmon-backup`, `.new`, `/tmp/rsmon-worker-act.*`, or lock leftovers. +- `TestSourceInstallActivationFailureRollback` (Alpine) forces four rerun + failures and proves each restores the prior install byte-for-byte and + running: a dirty checkout (build failure), a sabotaged atomic swap + (activation failure), `WORKER_CLUSTER_ENABLED=true` without credentials (the + new process exits at boot; start failure), and `WORKER_HOST=255.255.255.255` + (the new process runs but `/healthz` is unreachable; health failure). In + every case the installed binary SHA-256, env, service definition, single + process, and liveness match the pre-failure state, and no backup, lock, or + temp files leak. +- `TestSourceInstallActivationNoStart` installs the full layout without + starting anything, and reruns stay idle. +- The targeted shell fixture tests (`TestActivateShell*`) execute the real + activation script against stub `systemctl`/`rc-service`/`rc-update` + implementations and cover the init-managed paths the Docker fixtures cannot + reach: systemd unit install/enable with a `systemctl is-active`-based health + loop (and no pid file), OpenRC install/enable with an `rc-service`-based + health loop, fresh-failure rollback disabling a newly-enabled unit, rerun + rollback restoring the prior unit and its enable state, stale-lock breaking, + and interrupted-run recovery from a leftover backup marker. +- The unit suite covers the per-step remote scripts, the activation script + (backup + recovery marker, activation lock, atomic install, rollback trap, + supervisor, pid/zombie handling, pid-belongs-to-binary checks, enable-state + restoration, secrets absent), OpenRC unit rendering (including env + export), option validation, branch/commit parsing, sudo wrapping (password + never in the command), and an in-process real-SSH orchestration flow with + failure paths for missing pinned branches, build failures, detection + failures, and activation/start/health failures. +- **Residual limitation (stated honestly):** live systemd and OpenRC cannot + run inside the Docker/OpenSSH harness (the container PID 1 is sshd), so the + init-managed supervisor paths are verified by the stub-based shell fixture + tests above rather than against a real init. The stub tools simulate unit + state and command flow, not real systemd/OpenRC unit semantics; a real + init-system smoke test on a booted host remains a follow-up. diff --git a/internal/installer/harness/integration_activation_test.go b/internal/installer/harness/integration_activation_test.go new file mode 100644 index 0000000..b8c8531 --- /dev/null +++ b/internal/installer/harness/integration_activation_test.go @@ -0,0 +1,390 @@ +package harness + +import ( + "bytes" + "context" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "golang.org/x/crypto/ssh" + + "rocketgit.ru/rsmon/worker/internal/installer" + "rocketgit.ru/rsmon/worker/internal/sshinstall" +) + +// testWorkerToken is the operator-supplied worker token used by the +// activation E2E tests. The fixture has no reachable control plane, but +// the worker still binds its webapp and serves /healthz, which is what +// activation verifies. +const testWorkerToken = "e2e-activation-test-token" + +// activationOptions builds the work-package-4 source-install options for +// a started fixture: full activation with a fixed test token, honoring +// the RSMON_TEST_SOURCE_REPO/BRANCH overrides the staging tests use. +func activationOptions(h *Harness, f Fixture) installer.SourceInstallOptions { + opts := installer.SourceInstallOptions{ + SSHOptions: installer.SSHOptions{ + Host: "127.0.0.1", + Port: h.Port(), + User: f.UserOrDefault(), + IdentityFile: testKeyPath(), + KnownHostsFile: h.KnownHostsPath(), + }, + Activation: installer.ActivationOptions{ + Activate: true, + URL: "https://rsmon.ru", + Token: testWorkerToken, + }, + } + if repo := strings.TrimSpace(os.Getenv("RSMON_TEST_SOURCE_REPO")); repo != "" { + opts.Repo = repo + } + if branch := strings.TrimSpace(os.Getenv("RSMON_TEST_SOURCE_BRANCH")); branch != "" { + opts.Branch = branch + } + return opts +} + +// fixtureByName returns the fixture entry for a fixture name. +func fixtureByName(t *testing.T, name string) Fixture { + t.Helper() + for _, f := range Fixtures() { + if f.Name == name { + return f + } + } + t.Fatalf("no fixture named %q", name) + return Fixture{} +} + +// startActivatedFixture starts a fixture container and runs a full +// source install with activation against it, returning the harness, the +// options (so a test can rerun with modifications), the result, and a +// live SSH client. The caller owns client.Close and h.Stop (registered as +// a test cleanup). +func startActivatedFixture(t *testing.T, name string) (*Harness, installer.SourceInstallOptions, *installer.SourceInstallResult, *ssh.Client) { + t.Helper() + f := fixtureByName(t, name) + h, err := New("activate-"+name, f) + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute) + defer cancel() + if err := h.Start(ctx); err != nil { + t.Fatalf("start %s fixture: %v", name, err) + } + t.Cleanup(func() { + if err := h.Stop(); err != nil { + t.Errorf("stop %s fixture: %v", name, err) + } + }) + opts := activationOptions(h, f) + res, err := installer.SourceInstall(opts) + if err != nil { + t.Fatalf("activated source install on %s: %v", name, err) + } + client, err := h.Dial() + if err != nil { + t.Fatalf("dial %s after install: %v", name, err) + } + return h, opts, res, client +} + +// countWorkerProcesses returns the number of running processes whose +// command line is exactly the installed worker binary. The anchored -f +// pattern matches both busybox and procps pgrep (busybox's -x matches the +// full argv, so -x cannot be used portably). +func countWorkerProcesses(t *testing.T, client *ssh.Client, binary string) string { + t.Helper() + out, err := RunCommand(client, "pgrep -f '^"+binary+"$' | wc -l") + if err != nil { + t.Fatalf("count worker processes: %v", err) + } + return strings.TrimSpace(string(out)) +} + +// assertActivatedService verifies the full work-package-4 remote state: +// the installed layout and permissions, the native service definition for +// the detected init, exactly one running worker process, the pid file, +// and a live /healthz (via the installed binary's own liveness +// subcommand). +func assertActivatedService(t *testing.T, client *ssh.Client, name string, res *installer.SourceInstallResult) { + t.Helper() + a := res.Activation + if a == nil { + t.Fatalf("%s: activation result missing", name) + } + + out, err := RunCommand(client, "test -x "+shellQuote(a.Binary)+" && echo BIN_OK") + if err != nil || !strings.Contains(string(out), "BIN_OK") { + t.Fatalf("%s: installed binary not present/executable at %s: %q, %v", name, a.Binary, out, err) + } + + envMode, err := RunCommand(client, "stat -c '%a' "+shellQuote(a.EnvFile)) + if err != nil || strings.TrimSpace(string(envMode)) != "600" { + t.Fatalf("%s: env file mode = %q, want 600 (%v)", name, envMode, err) + } + cfgMode, err := RunCommand(client, "stat -c '%a' "+shellQuote(a.ConfigDir)) + if err != nil || strings.TrimSpace(string(cfgMode)) != "750" { + t.Fatalf("%s: config dir mode = %q, want 750 (%v)", name, cfgMode, err) + } + if _, err := RunCommand(client, "test -d "+shellQuote(a.DataDir)+"/webapp && echo DATA_OK"); err != nil { + t.Fatalf("%s: data dir not created at %s: %v", name, a.DataDir, err) + } + + // The fixtures run no init system (sshd is PID 1), so activation + // must use the embedded supervisor; the native service definition is + // still installed for real hosts. + if a.Supervisor != "none" { + t.Fatalf("%s: supervisor = %q, want none in the fixture", name, a.Supervisor) + } + unitMode := "644" + if res.Detection.InitSystem == sshinstall.InitOpenRC { + unitMode = "755" + } + if a.UnitFile == "" { + t.Fatalf("%s: no service definition installed", name) + } + if _, err := RunCommand(client, "test -f "+shellQuote(a.UnitFile)+" && echo UNIT_OK"); err != nil { + t.Fatalf("%s: unit file missing at %s: %v", name, a.UnitFile, err) + } + gotUnitMode, err := RunCommand(client, "stat -c '%a' "+shellQuote(a.UnitFile)) + if err != nil || strings.TrimSpace(string(gotUnitMode)) != unitMode { + t.Fatalf("%s: unit mode = %q, want %s (%v)", name, gotUnitMode, unitMode, err) + } + + // Exactly one worker process and a live /healthz. + if got := countWorkerProcesses(t, client, a.Binary); got != "1" { + t.Fatalf("%s: worker process count = %q, want 1", name, got) + } + if out, err := RunCommand(client, shellQuote(a.Binary)+" liveness"); err != nil || !strings.Contains(string(out), "ok") { + t.Fatalf("%s: installed worker /healthz check failed: %q, %v", name, out, err) + } + pidOut, err := RunCommand(client, "cat "+shellQuote(a.DataDir)+"/worker.pid") + if err != nil { + t.Fatalf("%s: pid file missing: %v", name, err) + } + if _, err := RunCommand(client, "kill -0 "+strings.TrimSpace(string(pidOut))+" && echo PID_OK"); err != nil { + t.Fatalf("%s: pid %s not alive: %v", name, pidOut, err) + } +} + +// assertNoActivationLeaks verifies a rerun left no backup dir, no lock, +// no /tmp upload leftovers, and no extra worker processes. +func assertNoActivationLeaks(t *testing.T, client *ssh.Client, name string, dataDir string) { + t.Helper() + out, err := RunCommand(client, + "leak=0; [ -e "+shellQuote(dataDir)+"/.rsmon-backup ] && leak=1; "+ + "[ -d "+shellQuote(dataDir)+"/.rsmon-activate.lock ] && leak=1; "+ + "ls /tmp 2>/dev/null | grep -q '^rsmon-worker-act\\.' && leak=1; "+ + "[ \"$leak\" -eq 1 ] && echo LEAK || echo CLEAN") + if err != nil || strings.TrimSpace(string(out)) != "CLEAN" { + t.Fatalf("%s: activation leftovers after rerun: %q, %v", name, out, err) + } +} + +// installedBinarySHA captures the SHA-256 of the installed worker binary +// so failure tests can prove the prior binary is preserved. +func installedBinarySHA(t *testing.T, client *ssh.Client, a *installer.ActivationResult) []byte { + t.Helper() + out, err := RunCommand(client, "sha256sum "+shellQuote(a.Binary)) + if err != nil { + t.Fatal(err) + } + return bytes.TrimSpace(out) +} + +// TestSourceInstallActivationFixtures is the work-package-4 success and +// idempotency acceptance test: on every fixture a full source install +// ends with an atomically installed binary, a mode-0600 env file, the +// data dir, a service definition for the detected init, and a running +// verified worker. A rerun succeeds, keeps a single worker process, +// rewrites the env deterministically, and leaks no temp or backup files. +// +// Opt-in: set RSMON_TEST_DOCKER=1 (see make test-ssh). +func TestSourceInstallActivationFixtures(t *testing.T) { + SkipUnlessEnabled(t) + + for _, f := range Fixtures() { + f := f + t.Run(f.Name, func(t *testing.T) { + _, opts, res, client := startActivatedFixture(t, f.Name) + defer client.Close() //nolint:errcheck + assertActivatedService(t, client, f.Name, res) + + // Rerun: idempotent, single service, no leaks. + res2, err := installer.SourceInstall(opts) + if err != nil { + t.Fatalf("%s: activated rerun: %v", f.Name, err) + } + if res2.Activation == nil || res2.ResolvedCommit != res.ResolvedCommit { + t.Fatalf("%s: rerun lost activation/commit: %+v", f.Name, res2) + } + assertActivatedService(t, client, f.Name, res2) + assertNoActivationLeaks(t, client, f.Name, res2.Activation.DataDir) + + envAfter, err := RunCommand(client, "cat "+shellQuote(res2.Activation.EnvFile)) + if err != nil { + t.Fatalf("%s: read env after rerun: %v", f.Name, err) + } + if !strings.Contains(string(envAfter), "RSMON_TOKEN="+testWorkerToken) { + t.Fatalf("%s: env lost the worker token after rerun: %q", f.Name, envAfter) + } + }) + } +} + +// TestSourceInstallActivationFailureRollback is the work-package-4 +// rollback acceptance test. After a successful activated install on the +// Alpine fixture, each rerun is forced to fail at a different stage - +// build (dirty checkout), activation (atomic swap sabotaged), start (the +// worker process dies at boot), and health (/healthz unreachable) - and +// every failure must leave the previous working install running with the +// same binary, the same single process, and /healthz answering. +// +// Opt-in: set RSMON_TEST_DOCKER=1 (see make test-ssh). +func TestSourceInstallActivationFailureRollback(t *testing.T) { + SkipUnlessEnabled(t) + + const fixture = "alpine" + _, opts, res, client := startActivatedFixture(t, fixture) + defer client.Close() //nolint:errcheck + a := res.Activation + assertActivatedService(t, client, fixture, res) + + // 1. Build failure preserves the prior install. + // A dirty tracked tree makes the rerun's checkout refuse before + // the build, so neither staging nor the service changes. + if _, err := RunCommand(client, "git -C "+shellQuote(res.Plan.BuildDir)+" checkout -q master~1 -- Makefile"); err != nil { + t.Fatalf("dirty the working tree: %v", err) + } + before := installedBinarySHA(t, client, a) + _, err := installer.SourceInstall(opts) + if err == nil || !strings.Contains(err.Error(), "check out branch") { + t.Fatalf("rerun on dirty tree err = %v, want checkout failure", err) + } + assertActivatedService(t, client, fixture, res) + if !bytes.Equal(before, installedBinarySHA(t, client, a)) { + t.Fatalf("installed binary changed after build-failure rerun") + } + // Restore the tree so later reruns can check out again. + if _, err := RunCommand(client, "git -C "+shellQuote(res.Plan.BuildDir)+" checkout -q HEAD -- Makefile"); err != nil { + t.Fatalf("restore working tree: %v", err) + } + + // 2. Activation failure preserves the prior install. + // Pre-creating the atomic-swap temp path as a directory makes the + // binary install step fail mid-activation; the rollback trap must + // restore the previous binary/env and restart the prior service. + if _, err := RunCommand(client, "mkdir -p /usr/local/bin/rsmon-worker.new"); err != nil { + t.Fatalf("sabotage the atomic binary swap: %v", err) + } + _, err = installer.SourceInstall(opts) + if err == nil || !strings.Contains(err.Error(), "activate worker service") { + t.Fatalf("activation-failure rerun err = %v, want activate failure", err) + } + assertActivatedService(t, client, fixture, res) + if !bytes.Equal(before, installedBinarySHA(t, client, a)) { + t.Fatalf("installed binary changed after activation-failure rerun") + } + assertNoActivationLeaks(t, client, fixture, a.DataDir) + if _, err := RunCommand(client, "rm -rf /usr/local/bin/rsmon-worker.new"); err != nil { + t.Fatalf("clean the swap sabotage: %v", err) + } + + // 3. Start failure preserves the prior install. + // WORKER_CLUSTER_ENABLED=true without WORKER_LOGIN/PASSWORD makes + // the new worker exit at boot (cluster init is fatal), so the + // activation supervisor detects the process died and rolls back. + envPath := filepath.Join(t.TempDir(), "worker.env") + startEnv := "RSMON_URL=https://rsmon.ru\nRSMON_TOKEN=" + testWorkerToken + "\nWORKER_CLUSTER_ENABLED=true\n" + if err := os.WriteFile(envPath, []byte(startEnv), 0o600); err != nil { + t.Fatal(err) + } + startOpts := opts + startOpts.Activation.EnvFile = envPath + _, err = installer.SourceInstall(startOpts) + if err == nil || !strings.Contains(err.Error(), "start failure") { + t.Fatalf("start-failure rerun err = %v, want start failure", err) + } + assertActivatedService(t, client, fixture, res) + if !bytes.Equal(before, installedBinarySHA(t, client, a)) { + t.Fatalf("installed binary changed after start-failure rerun") + } + assertNoActivationLeaks(t, client, fixture, a.DataDir) + + // 4. Health failure preserves the prior install. + // WORKER_HOST=255.255.255.255 makes the new worker bind fail and + // /healthz unreachable; the process stays up but the health gate + // fails and the rollback restores the prior loopback deployment. + healthOpts := opts + healthOpts.Activation.Host = "255.255.255.255" + _, err = installer.SourceInstall(healthOpts) + if err == nil || !strings.Contains(err.Error(), "health failure") { + t.Fatalf("health-failure rerun err = %v, want health failure", err) + } + assertActivatedService(t, client, fixture, res) + if !bytes.Equal(before, installedBinarySHA(t, client, a)) { + t.Fatalf("installed binary changed after health-failure rerun") + } + assertNoActivationLeaks(t, client, fixture, a.DataDir) +} + +// TestSourceInstallActivationNoStart installs the full layout (binary, +// env, data dir, service definition) without starting the worker, and +// asserts nothing runs afterward. Reruns stay idempotent. +// +// Opt-in: set RSMON_TEST_DOCKER=1 (see make test-ssh). +func TestSourceInstallActivationNoStart(t *testing.T) { + SkipUnlessEnabled(t) + + const fixture = "alpine" + f := fixtureByName(t, fixture) + h, err := New("nostart-"+fixture, f) + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute) + defer cancel() + if err := h.Start(ctx); err != nil { + t.Fatalf("start %s fixture: %v", fixture, err) + } + t.Cleanup(func() { + if err := h.Stop(); err != nil { + t.Errorf("stop: %v", err) + } + }) + opts := activationOptions(h, f) + opts.Activation.NoStart = true + res, err := installer.SourceInstall(opts) + if err != nil { + t.Fatalf("no-start install: %v", err) + } + client, err := h.Dial() + if err != nil { + t.Fatalf("dial: %v", err) + } + defer client.Close() //nolint:errcheck + a := res.Activation + if a == nil || a.Started { + t.Fatalf("no-start activation result = %+v", a) + } + if out, err := RunCommand(client, "test -x "+shellQuote(a.Binary)+" && test -f "+shellQuote(a.EnvFile)+" && echo INSTALLED"); err != nil || !strings.Contains(string(out), "INSTALLED") { + t.Fatalf("no-start did not install the layout: %q, %v", out, err) + } + if got := countWorkerProcesses(t, client, a.Binary); got != "0" { + t.Fatalf("no-start left a running worker: %q", got) + } + // A rerun is idempotent and still starts nothing. + if _, err := installer.SourceInstall(opts); err != nil { + t.Fatalf("no-start rerun: %v", err) + } + if got := countWorkerProcesses(t, client, a.Binary); got != "0" { + t.Fatalf("no-start rerun left a running worker: %q", got) + } + assertNoActivationLeaks(t, client, fixture, a.DataDir) +} diff --git a/internal/installer/harness/integration_test.go b/internal/installer/harness/integration_test.go index 3ff8c5f..407019c 100644 --- a/internal/installer/harness/integration_test.go +++ b/internal/installer/harness/integration_test.go @@ -325,6 +325,9 @@ func TestSourceInstallFixtures(t *testing.T) { IdentityFile: testKeyPath(), KnownHostsFile: h.KnownHostsPath(), }, + // Staging-only: service activation (work package 4) is + // exercised by the dedicated activation tests. + Activation: installer.ActivationOptions{Activate: false}, } if repo := strings.TrimSpace(os.Getenv("RSMON_TEST_SOURCE_REPO")); repo != "" { opts.Repo = repo @@ -446,6 +449,7 @@ func TestSourceInstallDirtyCheckoutPreservesStaging(t *testing.T) { IdentityFile: testKeyPath(), KnownHostsFile: h.KnownHostsPath(), }, + Activation: installer.ActivationOptions{Activate: false}, } if repo := strings.TrimSpace(os.Getenv("RSMON_TEST_SOURCE_REPO")); repo != "" { opts.Repo = repo diff --git a/internal/installer/install.go b/internal/installer/install.go index 1e25c79..e391162 100644 --- a/internal/installer/install.go +++ b/internal/installer/install.go @@ -478,6 +478,16 @@ func ValidateEnvironmentFile(path string) error { if err != nil { return fmt.Errorf("read worker environment: %w", err) } + _, err = parseEnvironmentContent(data) + return err +} + +// parseEnvironmentContent validates systemd-safe KEY=VALUE environment +// content and returns the parsed map. Values must not use quoting, +// interpolation, or whitespace; RSMON_URL and RSMON_TOKEN are required and +// must not be duplicated. Operating on the already-read bytes (rather +// than a path) keeps callers free of read-to-validate TOCTOU races. +func parseEnvironmentContent(data []byte) (map[string]string, error) { values := make(map[string]string) seenRequired := make(map[string]bool) for number, line := range strings.Split(string(data), "\n") { @@ -486,30 +496,30 @@ func ValidateEnvironmentFile(path string) error { continue } if strings.ContainsRune(line, '\r') { - return fmt.Errorf("worker environment line %d contains a carriage return", lineNumber) + return nil, fmt.Errorf("worker environment line %d contains a carriage return", lineNumber) } key, value, ok := strings.Cut(line, "=") if !ok || !envKeyPattern.MatchString(key) { - return fmt.Errorf("worker environment line %d must use KEY=VALUE syntax", lineNumber) + return nil, fmt.Errorf("worker environment line %d must use KEY=VALUE syntax", lineNumber) } if err := validateEnvValue(key, value); err != nil { - return fmt.Errorf("worker environment line %d: %w", lineNumber, err) + return nil, fmt.Errorf("worker environment line %d: %w", lineNumber, err) } if key == "RSMON_URL" || key == "RSMON_TOKEN" { if seenRequired[key] { - return fmt.Errorf("worker environment line %d duplicates %s", lineNumber, key) + return nil, fmt.Errorf("worker environment line %d duplicates %s", lineNumber, key) } seenRequired[key] = true } values[key] = value } if err := ValidateURL(values["RSMON_URL"]); err != nil { - return err + return nil, err } if err := ValidateToken(values["RSMON_TOKEN"]); err != nil { - return err + return nil, err } - return nil + return values, nil } // validateEnvValue enforces the value rules shared by the strict env diff --git a/internal/installer/sourceactivate.go b/internal/installer/sourceactivate.go new file mode 100644 index 0000000..2c5f0cf --- /dev/null +++ b/internal/installer/sourceactivate.go @@ -0,0 +1,769 @@ +package installer + +import ( + "errors" + "fmt" + "os" + "strings" + + "rocketgit.ru/rsmon/worker/internal/sshinstall" +) + +// ActivationOptions drives work package 4 of docs/source-installation.md: +// the atomic activation of the staged worker binary. It reuses the +// classic installer's environment resolution (resolveInstallEnv / +// renderEnvFile), on-disk layout (resolvePaths), and hardened systemd +// unit renderer, then installs the binary, env file, data dir, and the +// detected init's service definition atomically, starts/restarts the +// worker, and verifies the process and /healthz before declaring the +// activation a success. Any activation/start/health failure rolls back to +// the previous working install. +type ActivationOptions struct { + // Activate performs the atomic install + service activation after the + // staging build. When false SourceInstall stops at the staging build + // (the work-package-3 boundary) and touches no service configuration. + Activate bool + + // Name is the instance name ("" = the primary rsmon-worker service). + // A named instance gets rsmon-worker- paths and its own unit, + // matching the classic installer's multi-instance layout. + Name string + // URL is RSMON_URL (default https://rsmon.ru). + URL string + // Token is the worker API token (RSMON_TOKEN). Required for + // activation; prefer supplying it through a file at the CLI layer so + // it never appears in the process list. + Token string + // PublicURL is the advertised public origin (PUBLIC_URL). + PublicURL string + // Host is the worker webapp bind address (WORKER_HOST; default + // 127.0.0.1 as resolved by the classic installer). + Host string + // Port is the worker webapp bind port (WORKER_PORT; default 27401, + // required for named instances). + Port string + // Login / Password are the operator-console basic-auth credentials + // (WORKER_LOGIN / WORKER_PASSWORD). Both must be set or both empty. + Login string + Password string + // EnvFile is a local systemd-safe env file uploaded instead of the + // individual knobs. Validated with the classic strict parser. + EnvFile string + // NoStart installs the binary, env, data dir, and service definition + // without starting or restarting the worker. + NoStart bool +} + +// ActivationResult records where and how the staged worker was activated. +type ActivationResult struct { + Binary string // installed binary path + ConfigDir string + EnvFile string + DataDir string + UnitFile string // empty when no service definition was installed + UnitName string // systemd unit / rc-service name + Supervisor string // "systemd", "openrc", or "supervisor" (embedded fallback) + Started bool +} + +// installOptions projects the activation knobs onto the classic +// installer's option shape so the shared env resolution is reused. +func (a ActivationOptions) installOptions() InstallOptions { + return InstallOptions{ + URL: a.URL, + Token: a.Token, + PublicURL: a.PublicURL, + Host: a.Host, + Port: a.Port, + Login: a.Login, + Password: a.Password, + Name: strings.TrimSpace(a.Name), + NoStart: a.NoStart, + EnvFile: a.EnvFile, + } +} + +// renderEnv resolves the full worker environment exactly as the classic +// installer would and renders the canonical systemd-safe env file. The +// env file (when configured) is read exactly once and validated from the +// in-memory bytes, so a hostile local writer cannot swap the file +// between the read used for validation and the read used for the render. +func (a ActivationOptions) renderEnv() ([]byte, error) { + name := strings.TrimSpace(a.Name) + if err := validateInstanceName(name); err != nil { + return nil, err + } + var fileEnv map[string]string + if a.EnvFile != "" { + data, err := os.ReadFile(a.EnvFile) + if err != nil { + return nil, fmt.Errorf("--env-file: %w", err) + } + fe, perr := parseEnvironmentContent(data) + if perr != nil { + return nil, fmt.Errorf("--env-file: %w", perr) + } + fileEnv = fe + } + values, err := resolveInstallEnv(a.installOptions(), name, fileEnv) + if err != nil { + return nil, err + } + return renderEnvFile(values), nil +} + +// unitContent renders the service definition for the detected init +// system. systemd gets the classic hardened unit; OpenRC gets a native +// init script. An unknown init produces no unit (the no-service gate: +// the embedded supervisor still installs and manages the worker, but no +// native service definition is written). +func unitContent(init sshinstall.InitSystem, p paths) (content string, file string, mode string) { + svcName := strings.TrimSuffix(p.unitName, ".service") + switch init { + case sshinstall.InitSystemd: + return systemdUnitFor(p), p.unitFile, "0644" + case sshinstall.InitOpenRC: + return openrcInitFor(p), "/etc/init.d/" + svcName, "0755" + default: + return "", "", "" + } +} + +// runningInitScript reports the init system that is *actually running* +// (not merely installed): systemd, openrc, or none. The installer writes +// the service definition for the detected init but drives start/restart +// through whichever supervisor is usable right now; a container or chroot +// where no init runs falls back to the embedded supervisor. +const runningInitScript = `set -eu +if [ -d /run/systemd/system ] && command -v systemctl >/dev/null 2>&1; then + echo systemd +elif [ -e /run/openrc/softlevel ] && command -v rc-service >/dev/null 2>&1; then + echo openrc +else + echo none +fi` + +// normalizeSupervisor bounds the running-init probe output to the three +// supervisor kinds the activation script understands. +func normalizeSupervisor(raw string) string { + switch strings.TrimSpace(raw) { + case "systemd", "openrc": + return strings.TrimSpace(raw) + default: + return "none" + } +} + +// activateScript renders the single idempotent remote activation script. +// It is structured as a transaction over the previous install: +// +// 1. acquire an activation lock so concurrent runs cannot mutate the same +// deployment, and recover any interrupted prior activation from its +// leftover backup marker; +// 2. validate the staged binary (fails before any state is touched); +// 3. snapshot the prior binary/env/unit (and the unit's enable state) +// into a backup dir and write a recovery marker; +// 4. arm a rollback EXIT/HUP/INT/TERM trap that restores the snapshot, +// restores the prior enable state (or disables a freshly-installed +// unit on a fresh failure), and restarts the prior service; +// 5. install the new binary, env (mode 0600), data dir, and service +// definition atomically (temp file + rename); +// 6. start/restart through the active supervisor and verify the service +// stays active (systemd `is-active` / rc-service `status` / the +// embedded supervisor's zombie-aware pid check) and answers /healthz; +// 7. on success, drop the backup and release the lock; on failure the +// trap restores the prior install and the script exits non-zero. +// +// Reruns are idempotent: restart always stops the previous instance +// first, so exactly one worker process exists, and every temp file +// (backup dir, .new files, uploaded env/unit, lock) is removed on success +// and failure. +func activateScript(p activateParams) string { + r := strings.NewReplacer( + "@@STAGE@@", shellQuote(p.Stage), + "@@BINARY@@", shellQuote(p.Binary), + "@@CONFIG_DIR@@", shellQuote(p.ConfigDir), + "@@ENV_FILE@@", shellQuote(p.EnvFile), + "@@ENV_TMP@@", shellQuote(p.EnvTmp), + "@@DATA_DIR@@", shellQuote(p.DataDir), + "@@UNIT_TMP@@", shellQuote(p.UnitTmp), + "@@UNIT_FILE@@", shellQuote(p.UnitFile), + "@@UNIT_MODE@@", p.UnitMode, + "@@UNIT_NAME@@", shellQuote(p.UnitName), + "@@RC_NAME@@", shellQuote(p.RCName), + "@@SUPERVISOR@@", p.Supervisor, + "@@NO_START@@", p.NoStart, + ) + return r.Replace(activateTemplate) +} + +// activateTemplate is the remote script body. Sentinels (@@..@@) are +// substituted by activateScript; every operator-controlled value is +// single-quoted and validated on the controller side first. +const activateTemplate = `set -eu + +stage=@@STAGE@@ +binary=@@BINARY@@ +config_dir=@@CONFIG_DIR@@ +env_file=@@ENV_FILE@@ +env_tmp=@@ENV_TMP@@ +data_dir=@@DATA_DIR@@ +unit_tmp=@@UNIT_TMP@@ +unit_file=@@UNIT_FILE@@ +unit_mode=@@UNIT_MODE@@ +unit_name=@@UNIT_NAME@@ +rc_name=@@RC_NAME@@ +supervisor=@@SUPERVISOR@@ +no_start=@@NO_START@@ + +backup_dir="$data_dir/.rsmon-backup" +lock_dir="$data_dir/.rsmon-activate.lock" +pid_file="$data_dir/worker.pid" +log_file="$data_dir/worker.log" + +run_env() { + # Execute "$@" with a fresh environment built only from the env file + # plus PATH, HOME, and the webapp data dir. Sourcing the env file in a + # clean env -i shell prevents variables from a previous activation + # (for example WORKER_CLUSTER_ENABLED) from leaking into the new + # worker process after a rollback restores an older env file. + env -i PATH="/usr/bin:/bin:/sbin:/usr/sbin" HOME="$data_dir" \ + RSMON_WEBAPP_DATA_DIR="$data_dir/webapp" \ + RSMON_WORKER_ENV_FILE="$env_file" sh -c ' + set -a + . "$RSMON_WORKER_ENV_FILE" + set +a + unset RSMON_WORKER_ENV_FILE + exec "$@" + ' sh "$@" +} + +process_up() { + # True when the given pid is a live, non-zombie process. A zombie + # still answers kill -0 (its task struct exists until reaped), and in + # a container where PID 1 does not reap children a killed worker can + # linger as a zombie for a long time, so the /proc state must be + # checked explicitly. + [ -n "$1" ] || return 1 + kill -0 "$1" 2>/dev/null || return 1 + state="$(awk '{print $3}' "/proc/$1/stat" 2>/dev/null || true)" + [ "$state" != "Z" ] || return 1 +} + +process_is_worker() { + # True when the given pid's executable is (or was, before an atomic + # binary swap) the configured worker binary, so a stale or recycled + # pid file can never make the supervisor kill an unrelated process. + [ -n "$1" ] || return 1 + exe="$(readlink "/proc/$1/exe" 2>/dev/null || true)" + case "$exe" in + "$binary"|"$binary (deleted)") return 0 ;; + esac + return 1 +} + +stop_process() { + if [ -f "$pid_file" ]; then + pid="$(cat "$pid_file" 2>/dev/null || true)" + if [ -n "$pid" ] && process_is_worker "$pid" && process_up "$pid"; then + kill "$pid" 2>/dev/null || true + i=0 + while [ "$i" -lt 10 ]; do + if ! process_up "$pid"; then + break + fi + sleep 1 + i=$((i + 1)) + done + kill -9 "$pid" 2>/dev/null || true + fi + fi + rm -f "$pid_file" +} + +start_process() { + stop_process + mkdir -p "$data_dir" + run_env sh -c 'nohup "$1" >>"$2" 2>&1 & echo $! > "$3"' sh "$binary" "$log_file" "$pid_file" +} + +process_alive() { + [ -f "$pid_file" ] || return 1 + pid="$(cat "$pid_file" 2>/dev/null || true)" + process_is_worker "$pid" || return 1 + process_up "$pid" +} + +worker_come_up() { + # Wait up to 10s for the freshly-started worker to become a live, + # non-zombie process running the configured binary. The recorded pid + # starts as the nohup/sh child before it execs the worker, so early + # readlink checks can transiently see the interpreter; this retry both + # tolerates that exec window and catches genuine immediate deaths. + i=0 + while [ "$i" -lt 10 ]; do + if process_alive; then + return 0 + fi + sleep 1 + i=$((i + 1)) + done + return 1 +} + +health_ok() { + run_env "$binary" liveness >/dev/null 2>&1 +} + +svc_active() { + # The health loop verifies the service is active through the + # supervisor (never through a pid file for the init-managed paths). + case "$supervisor" in + systemd) systemctl is-active --quiet "$unit_name" ;; + openrc) rc-service "$rc_name" status >/dev/null 2>&1 ;; + none) process_alive ;; + esac +} + +stop_svc() { + case "$supervisor" in + systemd) systemctl stop "$unit_name" >/dev/null 2>&1 || true ;; + openrc) rc-service "$rc_name" stop >/dev/null 2>&1 || true ;; + none) stop_process ;; + esac +} + +restart_svc() { + case "$supervisor" in + systemd) + systemctl daemon-reload >/dev/null 2>&1 + if ! systemctl restart "$unit_name" >/dev/null 2>&1; then + echo "start failure: systemctl restart $unit_name failed" >&2 + return 1 + fi + if ! systemctl is-active --quiet "$unit_name"; then + echo "start failure: unit $unit_name is not active" >&2 + return 1 + fi + ;; + openrc) + if ! rc-service "$rc_name" restart >/dev/null 2>&1; then + echo "start failure: rc-service restart $rc_name failed" >&2 + return 1 + fi + if ! rc-service "$rc_name" status >/dev/null 2>&1; then + echo "start failure: service $rc_name is not running" >&2 + return 1 + fi + ;; + none) + start_process + if ! worker_come_up; then + echo "start failure: worker process did not stay alive" >&2 + return 1 + fi + ;; + esac + i=0 + while [ "$i" -lt 30 ]; do + if health_ok; then + return 0 + fi + # A worker that stops while the health gate is still probing is a + # start failure (it never became a stable service), not merely an + # unhealthy-but-running one. + if ! svc_active; then + echo "start failure: worker service is not active" >&2 + return 1 + fi + sleep 1 + i=$((i + 1)) + done + echo "health failure: worker did not answer /healthz" >&2 + return 1 +} + +# --- unit enable helpers ------------------------------------------------- +# unit_was_enabled records whether the unit referenced by the current +# params is enabled (best-effort; a missing init leaves it disabled). +unit_was_enabled() { + case "$supervisor" in + systemd) systemctl is-enabled "$unit_name" >/dev/null 2>&1 && unit_enabled=1 ;; + openrc) [ -e "/etc/runlevels/default/$rc_name" ] && unit_enabled=1 ;; + esac + return 0 +} +# set_unit_enabled "$1" enables (1) or disables (0) the unit. +set_unit_enabled() { + case "$supervisor" in + systemd) + systemctl daemon-reload >/dev/null 2>&1 || true + if [ "$1" -eq 1 ]; then + systemctl enable "$unit_name" >/dev/null 2>&1 || true + else + systemctl disable "$unit_name" >/dev/null 2>&1 || true + fi + ;; + openrc) + if [ "$1" -eq 1 ]; then + rc-update add "$rc_name" default >/dev/null 2>&1 || true + else + rc-update del "$rc_name" default >/dev/null 2>&1 || true + fi + ;; + esac + return 0 +} + +# --- activation lock ------------------------------------------------------ +# Prevent concurrent activations on the same host from racing the +# deployed-state mutation. A stale lock (left over from a SIGKILL'd run +# whose pid is no longer alive) is broken automatically. +acquire_lock() { + mkdir -p "$data_dir" + if [ -d "$lock_dir" ]; then + lockpid="$(cat "$lock_dir/pid" 2>/dev/null || true)" + if [ -n "$lockpid" ] && ! kill -0 "$lockpid" 2>/dev/null; then + rm -rf "$lock_dir" 2>/dev/null || true + fi + fi + i=0 + while [ "$i" -lt 60 ]; do + if mkdir "$lock_dir" 2>/dev/null; then + chmod 0700 "$lock_dir" 2>/dev/null || true + echo $$ > "$lock_dir/pid" 2>/dev/null || true + return 0 + fi + sleep 1 + i=$((i + 1)) + done + echo "another rsmon-worker activation is in progress ($lock_dir)" >&2 + return 1 +} +release_lock() { + rm -rf "$lock_dir" 2>/dev/null || true +} + +if ! acquire_lock; then + exit 1 +fi +# Any early failure (before the rollback trap is armed) still releases the +# lock. +trap 'release_lock' EXIT + +# --- interrupted-run recovery -------------------------------------------- +# A previous activation that was killed mid-flight (SSH drop, SIGKILL) +# leaves its backup marker behind. Restore that snapshot before the fresh +# activation runs, so the host never stays half-activated and the next run +# starts from a consistent prior state. +unit_enabled=0 +recover_backup() { + if [ ! -f "$backup_dir/.marker" ]; then + return 0 + fi + echo "recovering interrupted activation from $backup_dir" >&2 + if [ -f "$backup_dir/binary" ]; then + mkdir -p "$(dirname "$binary")" || true + cp -p "$backup_dir/binary" "$binary" || true + fi + if [ -f "$backup_dir/worker.env" ]; then + mkdir -p "$config_dir" || true + cp -p "$backup_dir/worker.env" "$env_file" || true + fi + if [ -n "$unit_file" ] && [ -f "$backup_dir/unit" ]; then + mkdir -p "$(dirname "$unit_file")" || true + cp -p "$backup_dir/unit" "$unit_file" || true + chmod "$unit_mode" "$unit_file" || true + fi + if [ "$(cat "$backup_dir/.marker" 2>/dev/null || true)" = "1" ]; then + unit_enabled=1 + fi + set_unit_enabled "$unit_enabled" || true + if [ -f "$backup_dir/binary" ] && [ -f "$backup_dir/worker.env" ] && [ "$no_start" -ne 1 ]; then + restart_svc || true + fi + rm -rf "$backup_dir" || true +} +recover_backup + +# --- staged binary validation (no state touched) ------------------------- +# A corrupt or missing staging build aborts here and leaves the prior +# install completely untouched (the rollback trap is not armed yet). +"$stage" --version >/dev/null + +# --- snapshot the prior install ------------------------------------------ +rm -rf "$backup_dir" +mkdir -p "$backup_dir" +had_binary=0 +had_env=0 +had_unit=0 +unit_enabled=0 +if [ -e "$binary" ]; then + cp -p "$binary" "$backup_dir/binary" + had_binary=1 +fi +if [ -e "$env_file" ]; then + cp -p "$env_file" "$backup_dir/worker.env" + had_env=1 +fi +if [ -n "$unit_file" ] && [ -e "$unit_file" ]; then + had_unit=1 + cp -p "$unit_file" "$backup_dir/unit" + unit_was_enabled +fi +# The marker is written only after the snapshot copies so recovery never +# sees a partial snapshot; it records whether the prior unit was enabled. +printf '%s\n' "$unit_enabled" > "$backup_dir/.marker" + +# --- rollback trap -------------------------------------------------------- +# Any failure from here on restores the snapshot (preserving binary/env/unit +# metadata), restores the prior enable state (or disables a freshly-installed +# unit on a fresh install), and brings the prior service back up. The trap +# also fires on HUP/INT/TERM so an interrupted run rolls back instead of +# leaving a half-activated state. +rolled_back=0 +rollback() { + [ "$rolled_back" -eq 1 ] && return 0 + rolled_back=1 + echo "rsmon-worker activation failed; restoring the prior install" >&2 + stop_svc || true + rm -f "$binary.new" "$env_file.new" "$unit_file.new" "$pid_file" || true + if [ "$had_binary" -eq 1 ]; then + cp -p "$backup_dir/binary" "$binary" || true + elif [ -e "$binary" ]; then + rm -f "$binary" || true + fi + if [ "$had_env" -eq 1 ]; then + cp -p "$backup_dir/worker.env" "$env_file" || true + chmod 0600 "$env_file" || true + elif [ -e "$env_file" ]; then + rm -f "$env_file" || true + fi + if [ -n "$unit_file" ]; then + if [ "$had_unit" -eq 1 ]; then + cp -p "$backup_dir/unit" "$unit_file" || true + chmod "$unit_mode" "$unit_file" || true + set_unit_enabled "$unit_enabled" || true + else + set_unit_enabled 0 || true + rm -f "$unit_file" || true + fi + fi + rm -f "$env_tmp" "$unit_tmp" || true + rm -rf "$backup_dir" || true + if [ "$had_binary" -eq 1 ] && [ "$had_env" -eq 1 ] && [ "$no_start" -ne 1 ]; then + restart_svc || true + fi + release_lock + exit 1 +} +trap rollback EXIT HUP INT TERM + +# --- atomic binary install ------------------------------------------------ +mkdir -p "$(dirname "$binary")" +install -m 0755 "$stage" "$binary.new" +mv -f "$binary.new" "$binary" + +# --- atomic env install (secrets, mode 0600) ------------------------------ +mkdir -p "$config_dir" +chmod 0750 "$config_dir" +install -m 0600 "$env_tmp" "$env_file.new" +mv -f "$env_file.new" "$env_file" +rm -f "$env_tmp" + +# --- data directory ------------------------------------------------------- +mkdir -p "$data_dir" "$data_dir/webapp" +chmod 0755 "$data_dir" "$data_dir/webapp" + +# --- service definition --------------------------------------------------- +if [ -n "$unit_file" ] && [ -f "$unit_tmp" ]; then + mkdir -p "$(dirname "$unit_file")" + install -m "$unit_mode" "$unit_tmp" "$unit_file.new" + mv -f "$unit_file.new" "$unit_file" + rm -f "$unit_tmp" + set_unit_enabled 1 +fi + +# --- start and verify ----------------------------------------------------- +# A failed start or a worker that does not answer /healthz triggers the +# rollback trap. +if [ "$no_start" -ne 1 ]; then + if ! restart_svc; then + echo "rsmon-worker activation failed: process or /healthz verification failed" >&2 + exit 1 + fi +else + stop_svc || true +fi + +# --- success -------------------------------------------------------------- +rm -rf "$backup_dir" +rm -f "$env_tmp" "$unit_tmp" +release_lock +trap - EXIT HUP INT TERM +if [ "$no_start" -eq 1 ]; then + echo "rsmon-worker activated: binary=$binary supervisor=$supervisor unit=${unit_file:-none} started=no" +else + echo "rsmon-worker activated: binary=$binary supervisor=$supervisor unit=${unit_file:-none} started=yes" +fi +exit 0 +` + +// activateParams are the resolved, controller-validated values fed into +// the remote activation script. +type activateParams struct { + Stage string + Binary string + ConfigDir string + EnvFile string + EnvTmp string + DataDir string + UnitTmp string + UnitFile string + UnitMode string + UnitName string // systemd unit name (rsmon-worker.service) + RCName string // rc-service name (rsmon-worker) + Supervisor string // systemd | openrc | none + NoStart string // 0 or 1 +} + +// activateWorker performs the work-package-4 activation for an already +// staged source build over the live SSH executor. It uses the env bytes +// rendered once during option normalization, uploads the env and service +// definition into a server-created 0700 temp dir (no /tmp symlink/TOCTOU +// attack surface), runs the atomic activation script, and records where +// and how the worker was activated. Every remote temp file it creates is +// removed on success and failure. +func (e *sourceExecutor) activateWorker(opts SourceInstallOptions, res *SourceInstallResult) error { + a := opts.Activation + name := strings.TrimSpace(a.Name) + p := resolvePaths(name) + + envData := opts.activationEnv + if len(envData) == 0 { + return errors.New("activation environment was not rendered (internal error)") + } + + unitContent, unitFile, unitMode := unitContent(res.Detection.InitSystem, p) + svcName := strings.TrimSuffix(p.unitName, ".service") + + supOut, err := e.runPlain("sh -c " + shellQuote(runningInitScript)) + if err != nil { + return fmt.Errorf("detect running init system: %w", err) + } + supervisor := normalizeSupervisor(string(supOut)) + + // Create a server-side 0700 temp dir owned by the SSH user, then + // upload the env/unit into it. mktemp -d produces an unpredictable, + // private path, so a hostile local user cannot pre-create a symlink at + // a predictable /tmp name (the classic upload TOCTOU). The token + // never appears in argv or logs: it travels only as base64 over the + // session stdin and later only inside the mode-0600 env file. + out, err := e.runPlain("d=$(mktemp -d /tmp/rsmon-worker-act.XXXXXX) && chmod 0700 \"$d\" && echo \"$d\"") + if err != nil { + return fmt.Errorf("create secure upload directory: %w", err) + } + secureDir := strings.TrimSpace(string(out)) + if secureDir == "" || !strings.HasPrefix(secureDir, "/tmp/") { + return errors.New("remote returned an invalid secure upload directory") + } + envTmp := secureDir + "/worker.env" + unitTmp := secureDir + "/unit" + if err := uploadBytes(e.client, envData, envTmp, 0o600); err != nil { + return fmt.Errorf("upload worker environment: %w", err) + } + if unitContent != "" { + if err := uploadBytes(e.client, []byte(unitContent), unitTmp, 0o644); err != nil { + return fmt.Errorf("upload service definition: %w", err) + } + } + defer func() { + _ = runRemote(e.client, "rm -rf -- "+shellQuote(secureDir), nil) + }() + + noStart := "0" + if a.NoStart { + noStart = "1" + } + script := activateScript(activateParams{ + Stage: res.StageBinary, + Binary: p.binary, + ConfigDir: p.configDir, + EnvFile: p.envFile, + EnvTmp: envTmp, + DataDir: p.dataDir, + UnitTmp: unitTmp, + UnitFile: unitFile, + UnitMode: unitMode, + UnitName: p.unitName, + RCName: svcName, + Supervisor: supervisor, + NoStart: noStart, + }) + if _, err := e.runPrivileged("sh -c " + shellQuote(script)); err != nil { + return fmt.Errorf("activate worker service: %w", err) + } + + res.Activation = &ActivationResult{ + Binary: p.binary, + ConfigDir: p.configDir, + EnvFile: p.envFile, + DataDir: p.dataDir, + UnitFile: unitFile, + UnitName: p.unitName, + Supervisor: supervisor, + Started: !a.NoStart, + } + return nil +} + +// validateRenderedEnv runs the strict environment-file parser over the +// rendered env bytes in memory so the exact bytes written remotely pass +// the same gate as a user-supplied file, without re-reading a file. +func validateRenderedEnv(data []byte) error { + _, err := parseEnvironmentContent(data) + return err +} + +// openrcInitFor renders the native OpenRC init script for an instance. It +// mirrors the hardening of the systemd unit (data dir, env file, webapp +// data dir) using OpenRC conventions; it is written for real hosts where +// openrc is the running init, while containers that cannot run openrc +// fall back to the embedded supervisor. +func openrcInitFor(p paths) string { + description := "RSMon distributed monitoring worker" + svcName := strings.TrimSuffix(p.unitName, ".service") + if p.name != "" { + description += " (" + p.name + ")" + } + return fmt.Sprintf(`#!/sbin/openrc-run +# Managed by the rsmon-worker source installer; do not edit by hand. + +name=%s +description=%s +command=%s +command_background=true +pidfile=%s/worker.pid +output_log=%s/worker.log +error_log=%s/worker.log + +depend() { + need net +} + +start_pre() { + checkpath --directory --mode 0755 --owner root:root %s %s/webapp + if [ -f %s ]; then + # OpenRC runs the command with the init script's environment, so + # the env-file variables must be exported (set -a) or the worker + # would start without them. + set -a + . %s + set +a + fi + export HOME=%s + export RSMON_WEBAPP_DATA_DIR=%s/webapp +} +`, svcName, description, p.binary, p.dataDir, p.dataDir, p.dataDir, + p.dataDir, p.dataDir, p.envFile, p.envFile, p.dataDir, p.dataDir) +} diff --git a/internal/installer/sourceactivate_shell_test.go b/internal/installer/sourceactivate_shell_test.go new file mode 100644 index 0000000..248a451 --- /dev/null +++ b/internal/installer/sourceactivate_shell_test.go @@ -0,0 +1,438 @@ +package installer + +import ( + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +// This file exercises the rendered activation script against stub +// systemd/OpenRC implementations ("targeted shell fixture tests"). Real +// systemd and OpenRC cannot run inside the Docker/OpenSSH harness (the +// container PID 1 is sshd), so these tests execute the actual remote +// script on the host with fake `systemctl`, `rc-service`, and `rc-update` +// binaries that simulate an init-managed host. This covers the unit +// install/enable, the supervisor-driven restart, the `is-active`-based +// health loop, the enable-state rollback, the activation lock, and the +// interrupted-run recovery paths that the E2E fixtures cannot reach. +// +// Residual limitation: the stub init tools verify command flow and state +// transitions, not the real systemd/OpenRC unit semantics; a real +// init-system smoke test remains out of scope for the container harness. + +// systemctlStub records every invocation and simulates unit state in a +// fake state directory: `is-active`/`is-enabled` consult marker files, +// `restart` marks the unit active (or fails when the fail-restart marker +// is present), and `enable`/`disable` toggle the enabled marker. +const systemctlStub = `#!/bin/sh +echo "$*" >> "$SYSCTL_LOG" +action="$1" +shift +unit="" +for a in "$@"; do + case "$a" in --*) continue ;; esac + unit="$a" + break +done +case "$action" in + is-active) [ -f "$FAKE_SYSTEMD_DIR/$unit.active" ] || exit 3 ;; + is-enabled) [ -f "$FAKE_SYSTEMD_DIR/$unit.enabled" ] || exit 1 ;; + enable) touch "$FAKE_SYSTEMD_DIR/$unit.enabled"; exit 0 ;; + disable) rm -f "$FAKE_SYSTEMD_DIR/$unit.enabled"; exit 0 ;; + daemon-reload) exit 0 ;; + restart) [ -f "$FAKE_SYSTEMD_DIR/fail-restart" ] && exit 1 + touch "$FAKE_SYSTEMD_DIR/$unit.active"; exit 0 ;; + stop) rm -f "$FAKE_SYSTEMD_DIR/$unit.active"; exit 0 ;; + *) exit 0 ;; +esac +` + +// rcServiceStub simulates OpenRC service state; rcUpdateStub simulates +// runlevel enablement. +const rcServiceStub = `#!/bin/sh +echo "$*" >> "$RC_LOG" +svc="$1" +action="$2" +case "$action" in + restart) [ -f "$FAKE_RC_DIR/fail-restart" ] && exit 1 + touch "$FAKE_RC_DIR/$svc.active"; exit 0 ;; + status) [ -f "$FAKE_RC_DIR/$svc.active" ] || exit 1 ;; + stop) rm -f "$FAKE_RC_DIR/$svc.active"; exit 0 ;; + *) exit 0 ;; +esac +` + +const rcUpdateStub = `#!/bin/sh +echo "$*" >> "$RCU_LOG" +case "$1" in + add) touch "$FAKE_RC_DIR/$2.enabled"; exit 0 ;; + del) rm -f "$FAKE_RC_DIR/$2.enabled"; exit 0 ;; + *) exit 0 ;; +esac +` + +// workerStub stands in for the built worker binary: it satisfies the +// `--version` validation and the `/healthz` liveness probe. +const workerStub = `#!/bin/sh +if [ "$1" = "--version" ]; then + echo "rsmon-worker version=dev commit=deadbeefdead buildDate=2026-01-01T00:00:00Z" + exit 0 +fi +if [ "$1" = "liveness" ]; then + exit 0 +fi +exit 1 +` + +// initShellFixture builds a temp "host" with stub init tools and a stub +// worker, plus the uploaded env/unit the activation script consumes. +type initShellFixture struct { + root string + binDir string + sysdDir string + rcDir string + + sysctlLog string + rcLog string + rcuLog string + + params activateParams +} + +func writeExec(t *testing.T, path, content string) { + t.Helper() + if err := os.WriteFile(path, []byte(content), 0o700); err != nil { + t.Fatal(err) + } +} + +func newInitShellFixture(t *testing.T, supervisor string) *initShellFixture { + t.Helper() + root := t.TempDir() + fx := &initShellFixture{ + root: root, + binDir: filepath.Join(root, "bin"), + sysdDir: filepath.Join(root, "sysd"), + rcDir: filepath.Join(root, "rc"), + sysctlLog: filepath.Join(root, "sysctl.log"), + rcLog: filepath.Join(root, "rc.log"), + rcuLog: filepath.Join(root, "rcu.log"), + } + for _, d := range []string{ + fx.binDir, fx.sysdDir, fx.rcDir, filepath.Join(root, "upload"), + filepath.Join(root, "usr/local/bin"), filepath.Join(root, "etc/rsmon-worker"), + } { + if err := os.MkdirAll(d, 0o755); err != nil { + t.Fatal(err) + } + } + writeExec(t, filepath.Join(fx.binDir, "systemctl"), systemctlStub) + writeExec(t, filepath.Join(fx.binDir, "rc-service"), rcServiceStub) + writeExec(t, filepath.Join(fx.binDir, "rc-update"), rcUpdateStub) + + stage := filepath.Join(root, "usr/local/bin/rsmon-worker.stage") + writeExec(t, stage, workerStub) + + envTmp := filepath.Join(root, "upload", "worker.env") + unitTmp := filepath.Join(root, "upload", "unit") + if err := os.WriteFile(envTmp, []byte( + "RSMON_URL=https://rsmon.ru\nRSMON_TOKEN=shell-test-token\nWORKER_HOST=127.0.0.1\nWORKER_PORT=27401\n", + ), 0o600); err != nil { + t.Fatal(err) + } + unitFile := filepath.Join(root, "etc/systemd/system/rsmon-worker.service") + unitMode := "0644" + if supervisor == "openrc" { + unitFile = filepath.Join(root, "etc/init.d/rsmon-worker") + unitMode = "0755" + if err := os.MkdirAll(filepath.Dir(unitFile), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(unitTmp, []byte("# fake openrc unit\n"), 0o755); err != nil { + t.Fatal(err) + } + } else { + if err := os.MkdirAll(filepath.Dir(unitFile), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(unitTmp, []byte("[Unit]\n# fake systemd unit\n"), 0o644); err != nil { + t.Fatal(err) + } + } + + dataDir := filepath.Join(root, "var/lib/rsmon-worker") + fx.params = activateParams{ + Stage: stage, + Binary: filepath.Join(root, "usr/local/bin/rsmon-worker"), + ConfigDir: filepath.Join(root, "etc/rsmon-worker"), + EnvFile: filepath.Join(root, "etc/rsmon-worker/worker.env"), + EnvTmp: envTmp, + DataDir: dataDir, + UnitTmp: unitTmp, + UnitFile: unitFile, + UnitMode: unitMode, + UnitName: "rsmon-worker.service", + RCName: "rsmon-worker", + Supervisor: supervisor, + NoStart: "0", + } + return fx +} + +// run executes the rendered activation script with the stub init tools +// first in PATH. +func (fx *initShellFixture) run(t *testing.T) (string, error) { + t.Helper() + script := activateScript(fx.params) + cmd := exec.Command("sh", "-c", script) + cmd.Env = append( + os.Environ(), + "PATH="+fx.binDir+":"+os.Getenv("PATH"), + "FAKE_SYSTEMD_DIR="+fx.sysdDir, + "FAKE_RC_DIR="+fx.rcDir, + "SYSCTL_LOG="+fx.sysctlLog, + "RC_LOG="+fx.rcLog, + "RCU_LOG="+fx.rcuLog, + ) + out, err := cmd.CombinedOutput() + return string(out), err +} + +func (fx *initShellFixture) log(t *testing.T, path string) string { + t.Helper() + b, err := os.ReadFile(path) + if err != nil { + return "" + } + return string(b) +} + +func (fx *initShellFixture) assertNoLeftovers(t *testing.T) { + t.Helper() + for _, name := range []string{".rsmon-backup", ".rsmon-activate.lock"} { + if _, err := os.Stat(filepath.Join(fx.params.DataDir, name)); !os.IsNotExist(err) { + t.Fatalf("%s left behind after activation", name) + } + } +} + +// TestActivateShellSystemd runs the real activation script against a stub +// systemd host and verifies the init-managed flow: the unit is installed +// and enabled, restart goes through systemctl, and the health loop uses +// `systemctl is-active` (no pid file is ever created). +func TestActivateShellSystemd(t *testing.T) { + fx := newInitShellFixture(t, "systemd") + out, err := fx.run(t) + if err != nil { + t.Fatalf("systemd activation failed: %v\n%s", err, out) + } + if !strings.Contains(out, "rsmon-worker activated") { + t.Fatalf("no activation success line:\n%s", out) + } + unit, err := os.ReadFile(fx.params.UnitFile) + if err != nil { + t.Fatalf("unit not installed: %v", err) + } + if !strings.Contains(string(unit), "[Unit]") { + t.Fatalf("installed unit has wrong content: %q", unit) + } + st, err := os.Stat(fx.params.EnvFile) + if err != nil { + t.Fatalf("env not installed: %v", err) + } + if st.Mode().Perm() != 0o600 { + t.Fatalf("env mode = %v, want 0600", st.Mode().Perm()) + } + log := fx.log(t, fx.sysctlLog) + for _, want := range []string{ + "enable rsmon-worker.service", + "restart rsmon-worker.service", + "is-active --quiet rsmon-worker.service", + } { + if !strings.Contains(log, want) { + t.Fatalf("systemctl log missing %q:\n%s", want, log) + } + } + // The health loop used systemctl is-active, never the embedded + // supervisor's pid file. + if _, err := os.Stat(filepath.Join(fx.params.DataDir, "worker.pid")); !os.IsNotExist(err) { + t.Fatalf("systemd activation created a pid file; the health loop must use systemctl is-active") + } + if _, err := os.Stat(filepath.Join(fx.sysdDir, "rsmon-worker.service.enabled")); err != nil { + t.Fatalf("unit not enabled: %v", err) + } + fx.assertNoLeftovers(t) +} + +// TestActivateShellOpenRC runs the real activation script against a stub +// OpenRC host: the init script is installed executable, the runlevel +// enablement and rc-service restart/status are used, and the health loop +// goes through rc-service (no pid file). +func TestActivateShellOpenRC(t *testing.T) { + fx := newInitShellFixture(t, "openrc") + out, err := fx.run(t) + if err != nil { + t.Fatalf("openrc activation failed: %v\n%s", err, out) + } + if !strings.Contains(out, "rsmon-worker activated") { + t.Fatalf("no activation success line:\n%s", out) + } + st, err := os.Stat(fx.params.UnitFile) + if err != nil { + t.Fatalf("openrc init not installed: %v", err) + } + if st.Mode().Perm() != 0o755 { + t.Fatalf("openrc init mode = %v, want 0755", st.Mode().Perm()) + } + rcLog := fx.log(t, fx.rcLog) + for _, want := range []string{"rsmon-worker restart", "rsmon-worker status"} { + if !strings.Contains(rcLog, want) { + t.Fatalf("rc-service log missing %q:\n%s", want, rcLog) + } + } + rcuLog := fx.log(t, fx.rcuLog) + if !strings.Contains(rcuLog, "add rsmon-worker default") { + t.Fatalf("rc-update add not issued:\n%s", rcuLog) + } + if _, err := os.Stat(filepath.Join(fx.rcDir, "rsmon-worker.enabled")); err != nil { + t.Fatalf("service not enabled in the runlevel: %v", err) + } + if _, err := os.Stat(filepath.Join(fx.params.DataDir, "worker.pid")); !os.IsNotExist(err) { + t.Fatalf("openrc activation created a pid file; the health loop must use rc-service status") + } + fx.assertNoLeftovers(t) +} + +// TestActivateShellSystemdRollbackFreshDisablesUnit proves a failed +// activation on a fresh host disables the newly-enabled unit and removes +// the unit file (no dangling enablement). +func TestActivateShellSystemdRollbackFreshDisablesUnit(t *testing.T) { + fx := newInitShellFixture(t, "systemd") + if err := os.WriteFile(filepath.Join(fx.sysdDir, "fail-restart"), []byte(""), 0o644); err != nil { + t.Fatal(err) + } + out, err := fx.run(t) + if err == nil { + t.Fatal("activation succeeded despite restart failure") + } + if !strings.Contains(out, "start failure") { + t.Fatalf("no start-failure classification:\n%s", out) + } + log := fx.log(t, fx.sysctlLog) + if !strings.Contains(log, "disable rsmon-worker.service") { + t.Fatalf("fresh rollback did not disable the unit:\n%s", log) + } + if _, err := os.Stat(fx.params.UnitFile); !os.IsNotExist(err) { + t.Fatal("unit file left behind after fresh rollback") + } + if _, err := os.Stat(filepath.Join(fx.sysdDir, "rsmon-worker.service.enabled")); !os.IsNotExist(err) { + t.Fatal("unit still enabled after fresh rollback") + } + fx.assertNoLeftovers(t) +} + +// TestActivateShellSystemdRollbackRestoresEnableState proves a failed +// activation on a host with a prior enabled unit restores the unit +// byte-for-byte and re-applies the prior enabled state. +func TestActivateShellSystemdRollbackRestoresEnableState(t *testing.T) { + fx := newInitShellFixture(t, "systemd") + if err := os.WriteFile(fx.params.UnitFile, []byte("PRIOR-UNIT-CONTENT"), 0o644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(fx.sysdDir, "rsmon-worker.service.enabled"), []byte(""), 0o644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(fx.sysdDir, "fail-restart"), []byte(""), 0o644); err != nil { + t.Fatal(err) + } + out, err := fx.run(t) + if err == nil { + t.Fatal("activation succeeded despite restart failure") + } + if !strings.Contains(out, "start failure") { + t.Fatalf("no start-failure classification:\n%s", out) + } + unit, err := os.ReadFile(fx.params.UnitFile) + if err != nil { + t.Fatal(err) + } + if string(unit) != "PRIOR-UNIT-CONTENT" { + t.Fatalf("prior unit not restored: %q", unit) + } + log := fx.log(t, fx.sysctlLog) + if !strings.Contains(log, "enable rsmon-worker.service") { + t.Fatalf("prior enabled state not re-applied:\n%s", log) + } + if _, err := os.Stat(filepath.Join(fx.sysdDir, "rsmon-worker.service.enabled")); err != nil { + t.Fatalf("unit not re-enabled after rollback: %v", err) + } + fx.assertNoLeftovers(t) +} + +// TestActivateShellStaleLockBroken proves a leftover lock from a killed +// run (dead pid) is broken automatically and the activation proceeds. +func TestActivateShellStaleLockBroken(t *testing.T) { + fx := newInitShellFixture(t, "systemd") + lockDir := filepath.Join(fx.params.DataDir, ".rsmon-activate.lock") + if err := os.MkdirAll(lockDir, 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(lockDir, "pid"), []byte("999999\n"), 0o644); err != nil { + t.Fatal(err) + } + out, err := fx.run(t) + if err != nil { + t.Fatalf("activation failed with a stale lock: %v\n%s", err, out) + } + if !strings.Contains(out, "rsmon-worker activated") { + t.Fatalf("no activation success line:\n%s", out) + } + if _, err := os.Stat(lockDir); !os.IsNotExist(err) { + t.Fatal("lock not released after activation") + } +} + +// TestActivateShellRecoveryFromInterruptedBackup proves the interrupted-run +// recovery: a leftover backup marker from a killed activation is restored +// before the fresh run, so a later validation failure still leaves the +// recovered prior install in place. +func TestActivateShellRecoveryFromInterruptedBackup(t *testing.T) { + fx := newInitShellFixture(t, "none") + backupDir := filepath.Join(fx.params.DataDir, ".rsmon-backup") + if err := os.MkdirAll(backupDir, 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(backupDir, "binary"), []byte("RECOVERED-BINARY"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(backupDir, "worker.env"), []byte("RSMON_URL=https://rsmon.ru\nRSMON_TOKEN=test\n"), 0o600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(backupDir, ".marker"), []byte("0"), 0o600); err != nil { + t.Fatal(err) + } + // Corrupt the stage so the fresh activation fails at validation (before + // the fresh snapshot), proving the recovered files were already in place. + if err := os.WriteFile(fx.params.Stage, []byte("not a script"), 0o644); err != nil { + t.Fatal(err) + } + out, err := fx.run(t) + if err == nil { + t.Fatal("activation succeeded with a corrupt stage") + } + if !strings.Contains(out, "recovering interrupted activation") { + t.Fatalf("recovery did not run:\n%s", out) + } + b, err := os.ReadFile(fx.params.Binary) + if err != nil { + t.Fatalf("recovered binary not restored: %v", err) + } + if string(b) != "RECOVERED-BINARY" { + t.Fatalf("recovered binary content = %q", b) + } + if _, err := os.Stat(backupDir); !os.IsNotExist(err) { + t.Fatal("backup not consumed by recovery") + } +} diff --git a/internal/installer/sourceactivate_test.go b/internal/installer/sourceactivate_test.go new file mode 100644 index 0000000..ec7e0fa --- /dev/null +++ b/internal/installer/sourceactivate_test.go @@ -0,0 +1,409 @@ +package installer + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "rocketgit.ru/rsmon/worker/internal/sshinstall" +) + +func TestActivateScriptMarkers(t *testing.T) { + script := activateScript(activateParams{ + Stage: "/opt/rsmon-worker-src/rsmon-worker", + Binary: "/usr/local/bin/rsmon-worker", + ConfigDir: "/etc/rsmon-worker", + EnvFile: "/etc/rsmon-worker/worker.env", + EnvTmp: "/tmp/rsmon-worker-abc.env", + DataDir: "/var/lib/rsmon-worker", + UnitTmp: "/tmp/rsmon-worker-abc.service", + UnitFile: "/etc/systemd/system/rsmon-worker.service", + UnitMode: "0644", + UnitName: "rsmon-worker.service", + RCName: "rsmon-worker", + Supervisor: "none", + NoStart: "0", + }) + for _, want := range []string{ + "set -eu", + // staged binary validated before any state is touched + "\"$stage\" --version >/dev/null", + // snapshot + rollback + "backup_dir=\"$data_dir/.rsmon-backup\"", + "rm -rf \"$backup_dir\"", + "cp -p \"$binary\" \"$backup_dir/binary\"", + "cp -p \"$env_file\" \"$backup_dir/worker.env\"", + "printf '%s\\n' \"$unit_enabled\" > \"$backup_dir/.marker\"", + "trap rollback EXIT HUP INT TERM", + "restoring the prior install", + // atomic installs with the right perms + "install -m 0755 \"$stage\" \"$binary.new\"", + "mv -f \"$binary.new\" \"$binary\"", + "install -m 0600 \"$env_tmp\" \"$env_file.new\"", + "mv -f \"$env_file.new\" \"$env_file\"", + "chmod 0750 \"$config_dir\"", + "mkdir -p \"$data_dir\" \"$data_dir/webapp\"", + "install -m \"$unit_mode\" \"$unit_tmp\" \"$unit_file.new\"", + "mv -f \"$unit_file.new\" \"$unit_file\"", + // supervisor + pid + health verification + "pid_file=\"$data_dir/worker.pid\"", + "env -i PATH=\"/usr/bin:/bin:/sbin:/usr/sbin\"", + "RSMON_WORKER_ENV_FILE=\"$env_file\"", + ". \"$RSMON_WORKER_ENV_FILE\"", + "exec \"$@\"", + "nohup \"$1\" >>\"$2\" 2>&1 & echo $! > \"$3\"", + "run_env \"$binary\" liveness >/dev/null 2>&1", + "while [ \"$i\" -lt 30 ]; do", + "rsmon-worker activated:", + // activation lock + interrupted-run recovery + "lock_dir=\"$data_dir/.rsmon-activate.lock\"", + "acquire_lock", + "another rsmon-worker activation is in progress", + "recovering interrupted activation from $backup_dir", + ".marker", + // pid belongs to the expected binary before kill + "readlink \"/proc/$1/exe\"", + "\"$binary (deleted)\"", + "process_is_worker \"$pid\"", + // rollback preserves metadata and enable state + "cp -p \"$backup_dir/binary\" \"$binary\"", + "cp -p \"$backup_dir/worker.env\" \"$env_file\"", + "set_unit_enabled \"$unit_enabled\"", + "systemctl is-enabled", + // temp/backup cleanup on success + "rm -rf \"$backup_dir\"", + "release_lock", + "trap - EXIT HUP INT TERM", + } { + if !strings.Contains(script, want) { + t.Fatalf("activate script missing %q:\n%s", want, script) + } + } + // The rollback must be armed only after the snapshot so a corrupt + // staging binary (validated first) never triggers a destructive + // rollback of the prior install. + if !strings.Contains(script, "\"$stage\" --version") || !strings.Contains(script, "trap rollback EXIT HUP INT TERM") { + t.Fatalf("activate script must validate the staged binary before arming rollback:\n%s", script) + } + stageCheck := strings.Index(script, "\"$stage\" --version") + trapIdx := strings.Index(script, "trap rollback EXIT HUP INT TERM") + if stageCheck < 0 || trapIdx < stageCheck { + t.Fatalf("staged-binary validation must precede the rollback trap:\n%s", script) + } +} + +func TestActivateScriptSystemdSupervisor(t *testing.T) { + script := activateScript(activateParams{ + Stage: "/opt/rsmon-worker-src/rsmon-worker", + Binary: "/usr/local/bin/rsmon-worker", + ConfigDir: "/etc/rsmon-worker", + EnvFile: "/etc/rsmon-worker/worker.env", + EnvTmp: "/tmp/e.env", + DataDir: "/var/lib/rsmon-worker", + UnitTmp: "/tmp/u.service", + UnitFile: "/etc/systemd/system/rsmon-worker.service", + UnitMode: "0644", + UnitName: "rsmon-worker.service", + RCName: "rsmon-worker", + Supervisor: "systemd", + NoStart: "0", + }) + for _, want := range []string{ + "supervisor=systemd", + "systemctl daemon-reload", + "systemctl restart \"$unit_name\"", + "systemctl is-active --quiet \"$unit_name\"", + "systemctl enable \"$unit_name\"", + "svc_active()", + "start failure: systemctl restart $unit_name failed", + "start failure: worker service is not active", + } { + if !strings.Contains(script, want) { + t.Fatalf("systemd supervisor missing %q:\n%s", want, script) + } + } + // The systemd health loop must use `systemctl is-active`, never the + // embedded supervisor's pid file. + if !strings.Contains(script, "systemd) systemctl is-active --quiet \"$unit_name\"") { + t.Fatalf("systemd health loop must use systemctl is-active:\n%s", script) + } +} + +func TestActivateScriptOpenRCSupervisor(t *testing.T) { + script := activateScript(activateParams{ + Stage: "/opt/rsmon-worker-src/rsmon-worker", + Binary: "/usr/local/bin/rsmon-worker", + ConfigDir: "/etc/rsmon-worker", + EnvFile: "/etc/rsmon-worker/worker.env", + EnvTmp: "/tmp/e.env", + DataDir: "/var/lib/rsmon-worker", + UnitTmp: "/tmp/u", + UnitFile: "/etc/init.d/rsmon-worker", + UnitMode: "0755", + UnitName: "rsmon-worker.service", + RCName: "rsmon-worker", + Supervisor: "openrc", + NoStart: "0", + }) + for _, want := range []string{ + "rc-service \"$rc_name\" restart", + "rc-service \"$rc_name\" status", + "rc-update add \"$rc_name\" default", + "start failure: rc-service restart $rc_name failed", + "rc-update del \"$rc_name\" default", + } { + if !strings.Contains(script, want) { + t.Fatalf("openrc supervisor missing %q:\n%s", want, script) + } + } +} + +func TestActivateScriptNoStartSkipsRestart(t *testing.T) { + script := activateScript(activateParams{ + Stage: "/opt/rsmon-worker-src/rsmon-worker", Binary: "/usr/local/bin/rsmon-worker", + ConfigDir: "/etc/rsmon-worker", EnvFile: "/etc/rsmon-worker/worker.env", EnvTmp: "/tmp/e.env", + DataDir: "/var/lib/rsmon-worker", UnitTmp: "", UnitFile: "", UnitMode: "", + UnitName: "rsmon-worker.service", RCName: "rsmon-worker", Supervisor: "none", NoStart: "1", + }) + if !strings.Contains(script, "[ \"$no_start\" -ne 1 ]") { + t.Fatalf("no-start gate missing:\n%s", script) + } + if !strings.Contains(script, "started=no") { + t.Fatalf("no-start summary missing:\n%s", script) + } +} + +func TestActivateScriptNeverContainsSecrets(t *testing.T) { + const secret = "super-secret-token-value" + params := activateParams{ + Stage: "/opt/rsmon-worker-src/rsmon-worker", Binary: "/usr/local/bin/rsmon-worker", + ConfigDir: "/etc/rsmon-worker", EnvFile: "/etc/rsmon-worker/worker.env", EnvTmp: "/tmp/e.env", + DataDir: "/var/lib/rsmon-worker", UnitTmp: "/tmp/u", UnitFile: "/etc/systemd/system/rsmon-worker.service", + UnitMode: "0644", UnitName: "rsmon-worker.service", RCName: "rsmon-worker", + Supervisor: "systemd", NoStart: "0", + } + script := activateScript(params) + if strings.Contains(script, secret) { + t.Fatalf("activate script contains a secret:\n%s", script) + } + // The env file is referenced by path; its contents (the secrets) are + // only sourced at runtime and never echoed. + for _, want := range []string{"RSMON_TOKEN=", secret} { + if strings.Contains(script, want) { + t.Fatalf("activate script must not embed env contents (%q):\n%s", want, script) + } + } +} + +func TestOpenRCInit(t *testing.T) { + unit := openrcInitFor(resolvePaths("")) + for _, want := range []string{ + "#!/sbin/openrc-run", + "name=rsmon-worker", + "description=RSMon distributed monitoring worker", + "command=/usr/local/bin/rsmon-worker", + "command_background=true", + "pidfile=/var/lib/rsmon-worker/worker.pid", + "output_log=/var/lib/rsmon-worker/worker.log", + "need net", + ". /etc/rsmon-worker/worker.env", + "set -a", + "set +a", + "RSMON_WEBAPP_DATA_DIR=/var/lib/rsmon-worker/webapp", + } { + if !strings.Contains(unit, want) { + t.Fatalf("openrc init missing %q:\n%s", want, unit) + } + } + named := openrcInitFor(resolvePaths("edge")) + for _, want := range []string{ + "name=rsmon-worker-edge", "command=/usr/local/bin/rsmon-worker-edge", + "pidfile=/var/lib/rsmon-worker-edge/worker.pid", "/etc/rsmon-worker-edge/worker.env", "(edge)", + } { + if !strings.Contains(named, want) { + t.Fatalf("named openrc init missing %q:\n%s", want, named) + } + } +} + +func TestRunningInitScript(t *testing.T) { + for _, want := range []string{"set -eu", "/run/systemd/system", "systemctl", "/run/openrc/softlevel", "rc-service", "echo none"} { + if !strings.Contains(runningInitScript, want) { + t.Fatalf("running-init script missing %q:\n%s", want, runningInitScript) + } + } +} + +func TestNormalizeSupervisor(t *testing.T) { + for _, in := range []string{"systemd", "openrc", "systemd\n", " openrc "} { + want := strings.TrimSpace(in) + if got := normalizeSupervisor(in); got != want { + t.Fatalf("normalizeSupervisor(%q) = %q, want %q", in, got, want) + } + } + for _, in := range []string{"", "none", "sysvinit", " "} { + if got := normalizeSupervisor(in); got != "none" { + t.Fatalf("normalizeSupervisor(%q) = %q, want none", in, got) + } + } +} + +func TestUnitContent(t *testing.T) { + p := resolvePaths("") + systemd, file, mode := unitContent(sshinstall.InitSystemd, p) + if systemd == "" || file != "/etc/systemd/system/rsmon-worker.service" || mode != "0644" { + t.Fatalf("systemd unit content = %q, %q, %q", systemd, file, mode) + } + if !strings.Contains(systemd, "ExecStart=/usr/local/bin/rsmon-worker\n") { + t.Fatalf("systemd unit not the classic hardened unit:\n%s", systemd) + } + openrc, file, mode := unitContent(sshinstall.InitOpenRC, p) + if openrc == "" || file != "/etc/init.d/rsmon-worker" || mode != "0755" { + t.Fatalf("openrc unit content = %q, %q, %q", openrc, file, mode) + } + if content, file, mode := unitContent(sshinstall.InitUnknown, p); content != "" || file != "" || mode != "" { + t.Fatalf("unknown-init unit content = %q, %q, %q, want empty (no-service gate)", content, file, mode) + } +} + +func TestRenderEnvActivation(t *testing.T) { + t.Setenv("RSMON_URL", "") + t.Setenv("RSMON_TOKEN", "") + t.Setenv("WORKER_HOST", "") + t.Setenv("WORKER_PORT", "") + t.Setenv("WORKER_LOGIN", "") + t.Setenv("WORKER_PASSWORD", "") + + data, err := (ActivationOptions{Activate: true, URL: "https://rsmon.ru", Token: "secret"}).renderEnv() + if err != nil { + t.Fatal(err) + } + for _, want := range []string{"RSMON_URL=https://rsmon.ru\n", "RSMON_TOKEN=secret\n", "WORKER_HOST=127.0.0.1\n", "WORKER_PORT=27401\n"} { + if !strings.Contains(string(data), want) { + t.Fatalf("rendered env missing %q:\n%s", want, data) + } + } + + // PUBLIC_URL canonicalization is reused from the classic installer. + data, err = (ActivationOptions{ + Activate: true, URL: "https://rsmon.ru", Token: "secret", + PublicURL: "https://worker.example.com", + }).renderEnv() + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(data), "PUBLIC_URL=https://worker.example.com\n") { + t.Fatalf("rendered env missing PUBLIC_URL:\n%s", data) + } +} + +func TestRenderEnvActivationRejectsMixedBasicAuth(t *testing.T) { + t.Setenv("RSMON_TOKEN", "") + if _, err := (ActivationOptions{ + Activate: true, URL: "https://rsmon.ru", Token: "secret", + Login: "admin", + }).renderEnv(); err == nil { + t.Fatal("login-only basic auth accepted") + } +} + +func TestRenderEnvActivationRequiresToken(t *testing.T) { + if _, err := (ActivationOptions{Activate: true, URL: "https://rsmon.ru"}).renderEnv(); err == nil { + t.Fatal("activation without a token accepted") + } + if _, err := (ActivationOptions{Activate: true, Token: "x", URL: "not-a-url"}).renderEnv(); err == nil { + t.Fatal("activation with a malformed URL accepted") + } + if _, err := (ActivationOptions{Activate: true, Token: "x", URL: "https://rsmon.ru", Name: "Bad_Name"}).renderEnv(); err == nil { + t.Fatal("activation with an invalid instance name accepted") + } + if _, err := (ActivationOptions{Activate: true, Token: "x", URL: "https://rsmon.ru", Name: "edge"}).renderEnv(); err == nil { + t.Fatal("named activation without WORKER_PORT accepted") + } +} + +// TestRenderEnvActivationReadsEnvFileOnce proves the env file is read a +// single time and validated from the in-memory bytes: swapping the file +// after the read cannot smuggle a different value into the render, and a +// malformed file fails on the read bytes. +func TestRenderEnvActivationReadsEnvFileOnce(t *testing.T) { + t.Setenv("RSMON_TOKEN", "") + path := filepath.Join(t.TempDir(), "worker.env") + if err := os.WriteFile(path, []byte("RSMON_URL=https://rsmon.ru\nRSMON_TOKEN=file-token\nWORKER_PORT=28888\n"), 0o600); err != nil { + t.Fatal(err) + } + data, err := (ActivationOptions{Activate: true, EnvFile: path}).renderEnv() + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(data), "RSMON_TOKEN=file-token\n") || !strings.Contains(string(data), "WORKER_PORT=28888\n") { + t.Fatalf("env file values not rendered:\n%s", data) + } + // A malformed file must be rejected from the same read. + if err := os.WriteFile(path, []byte("RSMON_URL=https://rsmon.ru\nRSMON_TOKEN=has space\n"), 0o600); err != nil { + t.Fatal(err) + } + if _, err := (ActivationOptions{Activate: true, EnvFile: path}).renderEnv(); err == nil { + t.Fatal("malformed env file accepted") + } +} + +// TestNormalizeSourceOptionsActivationRendersOnce verifies the rendered +// env is computed during normalization and reused, so a later call cannot +// re-read a changed env file. +func TestNormalizeSourceOptionsActivationRendersOnce(t *testing.T) { + t.Setenv("RSMON_TOKEN", "") + path := filepath.Join(t.TempDir(), "worker.env") + if err := os.WriteFile(path, []byte("RSMON_URL=https://rsmon.ru\nRSMON_TOKEN=once-token\n"), 0o600); err != nil { + t.Fatal(err) + } + base := SourceInstallOptions{ + SSHOptions: SSHOptions{Host: "h", User: "u"}, + Activation: ActivationOptions{Activate: true, EnvFile: path}, + } + norm, err := normalizeSourceOptions(base) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(norm.activationEnv), "RSMON_TOKEN=once-token\n") { + t.Fatalf("activation env not cached during normalization: %q", norm.activationEnv) + } + // Even after the file changes, the cached render is authoritative. + if err := os.WriteFile(path, []byte("RSMON_URL=https://evil.test\nRSMON_TOKEN=evil\n"), 0o600); err != nil { + t.Fatal(err) + } + if strings.Contains(string(norm.activationEnv), "evil") { + t.Fatalf("changed env file leaked into the cached render: %q", norm.activationEnv) + } +} + +func TestNormalizeSourceOptionsActivation(t *testing.T) { + t.Setenv("RSMON_TOKEN", "") + base := SourceInstallOptions{SSHOptions: SSHOptions{Host: "h", User: "u"}} + if _, err := normalizeSourceOptions(base); err != nil { + t.Fatalf("staging-only options must stay valid: %v", err) + } + act := base + act.Activation = ActivationOptions{Activate: true, URL: "https://rsmon.ru"} + if _, err := normalizeSourceOptions(act); err == nil || !strings.Contains(err.Error(), "RSMON_TOKEN") { + t.Fatalf("activation without a token error = %v", err) + } + ok := base + ok.Activation = ActivationOptions{Activate: true, URL: "https://rsmon.ru", Token: "secret"} + if _, err := normalizeSourceOptions(ok); err != nil { + t.Fatalf("valid activation rejected: %v", err) + } +} + +func TestValidateRenderedEnv(t *testing.T) { + if err := validateRenderedEnv([]byte("RSMON_URL=https://rsmon.ru\nRSMON_TOKEN=secret\n")); err != nil { + t.Fatal(err) + } + if err := validateRenderedEnv([]byte("RSMON_TOKEN=secret\n")); err == nil { + t.Fatal("env without RSMON_URL accepted") + } + if err := validateRenderedEnv([]byte("RSMON_URL=https://rsmon.ru\nRSMON_TOKEN=secret value\n")); err == nil { + t.Fatal("env with whitespace accepted") + } +} diff --git a/internal/installer/sourceinstall.go b/internal/installer/sourceinstall.go index 3b867ac..70d124a 100644 --- a/internal/installer/sourceinstall.go +++ b/internal/installer/sourceinstall.go @@ -50,11 +50,23 @@ type SourceInstallOptions struct { ToolchainDir string // StageBinary is where the built worker binary is written. It must // be absolute and defaults to /rsmon-worker. The running - // service and its config are NOT touched by this work package. + // service and its config are NOT touched until Activation runs. StageBinary string // SessionTimeout bounds each remote command. 0 uses the default // (30 minutes); the build step can legitimately run for minutes. SessionTimeout time.Duration + // Activation drives work package 4: after the staging build, the + // staged binary, validated environment, data dir, and the detected + // init's service definition are installed atomically and the worker + // is started and verified (process + /healthz). Any failure rolls + // back to the prior working install. Empty keeps SourceInstall at the + // staging boundary and touches no service configuration. + Activation ActivationOptions + + // activationEnv is the once-rendered activation environment, computed + // during option normalization so the env file is read and rendered + // exactly once per run (no TOCTOU between preflight and activation). + activationEnv []byte } // SourceInstallResult is what a source installation resolved to. The @@ -70,6 +82,10 @@ type SourceInstallResult struct { ResolvedCommit string RecordFile string StageBinary string + // Activation is set when the staged binary was atomically installed + // and the worker started/verified. It records the installed layout + // and the supervisor used. Nil when Activation was not requested. + Activation *ActivationResult } // defaultToolchainDir is the standard Go installation prefix. @@ -142,14 +158,20 @@ func (e *sourceExecutor) fileProber() sshinstall.FileProber { // branch, builds the worker to a staging path, and only then records the // resolved branch and commit. // -// The running service, its configuration, and its data directory are -// deliberately untouched: atomic activation and rollback are the next -// work package. Every remote step runs with the same privilege path as -// `deploy` (root, passwordless sudo, or sudo -S), every interpolated -// value is single-quoted, every step script fails closed (`set -eu` or -// explicit `&&`/retry), and every failure returns a bounded, actionable -// error. Each remote command is capped by SessionTimeout and its stdout -// is size-bounded. +// When Activation.Activate is set, the staged binary is then atomically +// installed together with the validated environment, data directory, and +// the detected init's service definition, and the worker is started and +// verified (process + /healthz); any activation/start/health failure +// rolls back to the prior working install. Without activation the +// running service, its configuration, and its data directory are +// deliberately untouched (the staging boundary of work package 3). +// +// Every remote step runs with the same privilege path as `deploy` (root, +// passwordless sudo, or sudo -S), every interpolated value is +// single-quoted, every step script fails closed (`set -eu` or explicit +// `&&`/retry), and every failure returns a bounded, actionable error. +// Each remote command is capped by SessionTimeout and its stdout is +// size-bounded. func SourceInstall(opts SourceInstallOptions) (*SourceInstallResult, error) { opts, err := normalizeSourceOptions(opts) if err != nil { @@ -276,6 +298,15 @@ func SourceInstall(opts SourceInstallOptions) (*SourceInstallResult, error) { if _, err := executor.runPrivileged("sh -c " + shellQuote(commitRecordScript(plan.BuildDir, branch, commit))); err != nil { return nil, fmt.Errorf("record resolved commit: %w", err) } + + // Work package 4: atomically install the staged build and activate + // the worker. On any failure the remote script restores the previous + // working install and this step returns a bounded error. + if opts.Activation.Activate { + if err := executor.activateWorker(opts, result); err != nil { + return nil, err + } + } return result, nil } @@ -316,6 +347,20 @@ func normalizeSourceOptions(o SourceInstallOptions) (SourceInstallOptions, error if o.SessionTimeout <= 0 { o.SessionTimeout = defaultSessionTimeout } + if o.Activation.Activate { + // Render (and validate) the activation environment exactly once + // here. The rendered bytes are reused at activation time, so the + // env file is read a single time and cannot change between the + // preflight and the remote install (env-file TOCTOU). + env, err := o.Activation.renderEnv() + if err != nil { + return o, fmt.Errorf("activation: %w", err) + } + if err := validateRenderedEnv(env); err != nil { + return o, fmt.Errorf("activation: %w", err) + } + o.activationEnv = env + } return o, nil } diff --git a/internal/installer/sourceinstall_ssh_test.go b/internal/installer/sourceinstall_ssh_test.go index a505621..5fc0b6c 100644 --- a/internal/installer/sourceinstall_ssh_test.go +++ b/internal/installer/sourceinstall_ssh_test.go @@ -4,6 +4,7 @@ import ( "crypto/ed25519" "crypto/rand" "fmt" + "io" "net" "strings" "sync" @@ -106,7 +107,18 @@ func (s *fakeSSHServer) handleConn(conn net.Conn, config *ssh.ServerConfig) { } go func() { defer channel.Close() + // Drain the client's stdin so uploads (which stream base64 + // over the session stdin) never block the channel window. + // The channel is not closed until the client's write side is + // exhausted, mirroring the real server behavior. + var drained sync.WaitGroup + drained.Add(1) + go func() { + defer drained.Done() + _, _ = io.Copy(io.Discard, channel) + }() s.handleSession(channel, requests) + drained.Wait() }() } } @@ -151,9 +163,10 @@ func (s *fakeSSHServer) execCommand(channel ssh.Channel, command string) { // defaultFakeExec simulates a minimal Linux host: it answers os-release, // uname, the init-marker file probe, origin URL, branch resolution, and -// rev-parse, and accepts every install step. A pinned-branch existence -// check (show-ref) fails by default so the missing-branch path is -// exercised without extra setup. +// rev-parse, accepts every install step, and simulates the work-package-4 +// activation (running-init probe, env/unit uploads, and the activate +// script succeeding). A pinned-branch existence check (show-ref) fails by +// default so the missing-branch path is exercised without extra setup. func defaultFakeExec(command string) (string, string, int) { switch { case strings.Contains(command, "cat /etc/os-release"): @@ -170,6 +183,18 @@ func defaultFakeExec(command string) (string, string, int) { return fakeCommitHex + "\n", "", 0 case strings.Contains(command, "show-ref"): return "", "branch not found", 1 + case strings.Contains(command, "softlevel"): + // running-init probe: no systemd/openrc is actually booted. + return "none\n", "", 0 + case strings.Contains(command, "mktemp -d /tmp/rsmon-worker-act"): + // server-side secure 0700 upload dir. + return "/tmp/rsmon-worker-act-fake\n", "", 0 + case strings.Contains(command, "base64 -d"): + // env/unit uploads succeed. + return "", "", 0 + case strings.Contains(command, "rsmon-worker activated"): + // the activate script completed successfully. + return "", "", 0 default: return "", "", 0 } @@ -184,9 +209,25 @@ func testSSHOptions(port int) SourceInstallOptions { Password: fakeSSHPassword, InsecureHostKey: true, }, + // Staging-only by default so the work-package-3 orchestration + // tests keep their historical shape; activation tests opt in. + Activation: ActivationOptions{Activate: false}, } } +// testActivationOptions wraps testSSHOptions with the credentials and +// activation flag needed to run the work-package-4 flow against the fake +// server. +func testActivationOptions(port int) SourceInstallOptions { + opts := testSSHOptions(port) + opts.Activation = ActivationOptions{ + Activate: true, + URL: "https://rsmon.ru", + Token: fakeSSHPassword + "-token", + } + return opts +} + func TestSourceInstallSSHFlow(t *testing.T) { srv := startFakeSSHServer(t, nil) res, err := SourceInstall(testSSHOptions(srv.Port())) @@ -349,3 +390,147 @@ func TestSourceInstallSSHCommandTimeout(t *testing.T) { t.Fatalf("err = %v, want command timeout", err) } } + +// TestSourceInstallSSHActivationFlow verifies the work-package-4 flow +// over a real SSH session: after the staging build and commit record, the +// installer probes the running init, uploads the env and service +// definition, runs the atomic activation script, and reports the installed +// layout. The secrets-absent contract holds: the worker token never +// reaches a remote command. +func TestSourceInstallSSHActivationFlow(t *testing.T) { + srv := startFakeSSHServer(t, nil) + res, err := SourceInstall(testActivationOptions(srv.Port())) + if err != nil { + t.Fatal(err) + } + if res.Activation == nil { + t.Fatal("activation result missing") + } + if res.Activation.Binary != "/usr/local/bin/rsmon-worker" || + res.Activation.EnvFile != "/etc/rsmon-worker/worker.env" || + res.Activation.DataDir != "/var/lib/rsmon-worker" || + res.Activation.UnitFile != "/etc/systemd/system/rsmon-worker.service" || + res.Activation.Supervisor != "none" || + !res.Activation.Started { + t.Fatalf("activation result = %+v", res.Activation) + } + + commands := srv.Commands() + var joined strings.Builder + for _, c := range commands { + joined.WriteString(c) + joined.WriteString("\n") + } + for _, want := range []string{ + "/run/systemd/system", + "softlevel", + "mktemp -d /tmp/rsmon-worker-act", + "umask 077; base64 -d >", + "rsmon-worker activated", + "install -m 0755", + "install -m 0600", + "chmod 0750", + "worker.pid", + "liveness", + "rm -rf --", + } { + if !strings.Contains(joined.String(), want) { + t.Fatalf("activation commands missing %q:\n%s", want, joined.String()) + } + } + + // Ordering: build and commit record, then a server-side secure upload + // dir is created, the env/unit are uploaded into it, and the activation + // script runs last. + idx := func(sub string) int { + for i, c := range commands { + if strings.Contains(c, sub) { + return i + } + } + t.Fatalf("command %q not found in %v", sub, commands) + return -1 + } + if idx("branch=%s") >= idx("mktemp -d /tmp/rsmon-worker-act") || + idx("mktemp -d /tmp/rsmon-worker-act") >= idx("base64 -d >") || + idx("base64 -d >") >= idx("rsmon-worker activated") { + t.Fatalf("activation order wrong: %v", commands) + } + if strings.Contains(joined.String(), fakeSSHPassword+"-token") { + t.Fatal("worker token leaked into a remote command") + } +} + +// TestSourceInstallSSHNoActivationSkipsService proves the staging +// boundary: without Activation the installer never probes the init system, +// uploads an env/unit, or runs the activation script. +func TestSourceInstallSSHNoActivationSkipsService(t *testing.T) { + srv := startFakeSSHServer(t, nil) + res, err := SourceInstall(testSSHOptions(srv.Port())) + if err != nil { + t.Fatal(err) + } + if res.Activation != nil { + t.Fatalf("activation must not run without the flag: %+v", res.Activation) + } + joined := strings.Join(srv.Commands(), "\n") + for _, forbidden := range []string{"softlevel", "base64 -d >", "rsmon-worker activated", "worker.pid"} { + if strings.Contains(joined, forbidden) { + t.Fatalf("staging-only flow ran activation step %q:\n%s", forbidden, joined) + } + } +} + +// TestSourceInstallSSHActivationRequiresToken verifies the credentials +// gate: activation without a token fails before any remote connection. +func TestSourceInstallSSHActivationRequiresToken(t *testing.T) { + srv := startFakeSSHServer(t, nil) + opts := testSSHOptions(srv.Port()) + opts.Activation = ActivationOptions{Activate: true, URL: "https://rsmon.ru"} + _, err := SourceInstall(opts) + if err == nil || !strings.Contains(err.Error(), "RSMON_TOKEN") { + t.Fatalf("err = %v, want token requirement", err) + } + if got := len(srv.Commands()); got != 0 { + t.Fatalf("commands ran before validation failed: %d", got) + } +} + +// TestSourceInstallSSHActivationFailure verifies the bounded failure +// path: a failing activate script surfaces as a labelled error with the +// remote stderr, and the uploaded temp env/unit are cleaned up by the +// controller defer regardless of the outcome. +func TestSourceInstallSSHActivationFailure(t *testing.T) { + srv := startFakeSSHServer(t, func(command string) (string, string, int) { + if strings.Contains(command, "rsmon-worker activated") { + return "", "activation exploded: disk full", 1 + } + return defaultFakeExec(command) + }) + _, err := SourceInstall(testActivationOptions(srv.Port())) + if err == nil || !strings.Contains(err.Error(), "activate worker service") || !strings.Contains(err.Error(), "disk full") { + t.Fatalf("err = %v, want bounded activation failure", err) + } + joined := strings.Join(srv.Commands(), "\n") + if !strings.Contains(joined, "rm -rf --") { + t.Fatalf("secure upload dir cleanup not issued after activation failure:\n%s", joined) + } +} + +// TestSourceInstallSSHActivationHealthFailure verifies the health gate +// surfaces as an activation failure and the temp files are still cleaned. +// The fake cannot run the shell script, so the failure is injected at the +// activate-command level; the real rollback semantics are covered by the +// Docker/OpenSSH E2E tests. +func TestSourceInstallSSHActivationHealthFailure(t *testing.T) { + srv := startFakeSSHServer(t, func(command string) (string, string, int) { + if strings.Contains(command, "rsmon-worker activated") { + return "", "health failure: worker did not answer /healthz", 1 + } + return defaultFakeExec(command) + }) + _, err := SourceInstall(testActivationOptions(srv.Port())) + if err == nil || !strings.Contains(err.Error(), "did not answer /healthz") { + t.Fatalf("err = %v, want /healthz verification failure", err) + } +}