feat(installer): activate source builds atomically
Все проверки выполнены успешно
CI / test (push) Successful in 4m30s
Docker / Build and publish worker image (push) Successful in 17m26s

Этот коммит содержится в:
Gleb Tv
2026-08-13 02:26:37 +03:00
родитель bd6070ee1f
Коммит 674a7d82bf
11 изменённых файлов: 2534 добавлений и 86 удалений

Просмотреть файл

@@ -183,11 +183,18 @@ A Go SSH source installer is implemented as the `source-install` subcommand: it
detects the remote distro and architecture over the existing SSH transport,
installs the minimal build prerequisites, downloads and SHA-256-verifies the
pinned Go 1.26 toolchain, clones/updates the public repository, records the
resolved branch and commit, and builds the worker to a staging path. It
deliberately stops before touching the running service or its configuration
(atomic activation and rollback are the next milestone). The pure
detection/planning layer and the Docker/OpenSSH test harness that accepts it
are implemented; see
resolved branch and commit, builds the worker to a staging path, and then
atomically activates the staged binary together with the validated environment,
data directory, and the detected init's service definition. It starts the
worker and verifies the process and `/healthz`; any activation, start, or
health failure rolls back to the previous working install (preserving the
prior binary/env/unit and its enable state), concurrent activations are
serialized by a lock, a run killed mid-flight is recovered from its
leftover backup marker on the next activation, and reruns are idempotent
(exactly one worker process, no leaked temp files). Pass `--no-activate`
to stop at the staging build, and `--no-start` to install the layout
without starting the worker. The pure detection/planning layer and the
Docker/OpenSSH test harness that accepts it are implemented; see
[`docs/source-installation.md`](docs/source-installation.md); run the live
fixture matrix with `make test-ssh`.
@@ -195,7 +202,8 @@ fixture matrix with `make test-ssh`.
./bin/rsmon-worker source-install \
--host worker.example.com \
--user deploy \
--identity-file ~/.ssh/id_ed25519
--identity-file ~/.ssh/id_ed25519 \
--token-file ./worker.token
```
By default the installer builds the remote's default branch and records what it
@@ -204,13 +212,21 @@ resolves to (the public repository currently publishes `master`). Pass
install fails before building. The repository must be an `https://` URL without
userinfo.
The built binary is left at `/opt/rsmon-worker-src/rsmon-worker` (override with
The built binary is staged at `/opt/rsmon-worker-src/rsmon-worker` (override with
`--build-dir` / `--stage-binary`), the toolchain at `/usr/local/go` (replaced
atomically: download, verify, stage, swap with rollback), and the resolved
branch and commit in `/opt/rsmon-worker-src/rsmon-worker.commit` (written only
after a successful build). The same SSH auth, secret-file, and host-key options
as `deploy` apply; no worker token is sent because the source install does not
configure a service. Prefer `--key-passphrase-file`, `--password-file`, and
after a successful build). Activation installs the binary to
`/usr/local/bin/rsmon-worker`, the environment to
`/etc/rsmon-worker/worker.env` (mode 0600), the data directory to
`/var/lib/rsmon-worker`, and a systemd unit or OpenRC init script for the
detected init (an embedded supervisor manages the process when no init is
running, as in containers). The env and unit are uploaded into a server-side
0700 `mktemp -d` directory so no local user can race a predictable `/tmp` path,
and the env file is read/rendered exactly once. The same SSH auth,
secret-file, and host-key options as `deploy` apply; the worker token is
supplied with `--token`/`--token-file` and written only to the mode-0600 env
file. Prefer `--key-passphrase-file`, `--password-file`, and
`--sudo-password-file` over their direct-flag equivalents: file options keep
secrets out of the process list and shell history.

Просмотреть файл

@@ -156,16 +156,21 @@ func secretValue(direct, path string) (string, error) {
return strings.TrimRight(string(b), "\r\n"), nil
}
// sourceInstallCommand drives the remote source installer (work package
// 3 of docs/source-installation.md): prerequisites, verified Go
// toolchain, clone/update, resolved branch/commit record, and a staging
// build. It reuses the deploy SSH options; it never touches the running
// service, configuration, or data directory on the remote host.
// sourceInstallCommand drives the remote source installer
// (docs/source-installation.md): prerequisites, verified Go toolchain,
// clone/update, resolved branch/commit record, a staging build, and (work
// package 4) the atomic activation of the staged binary, validated
// environment, data dir, and the detected init's service definition,
// followed by a start and a process + /healthz verification. It reuses
// the deploy SSH options. Any activation/start/health failure rolls back
// to the prior working install; --no-activate keeps the staging-only
// behavior.
func sourceInstallCommand(args []string) int {
fs := flag.NewFlagSet("source-install", flag.ContinueOnError)
fs.SetOutput(os.Stderr)
var opts installer.SourceInstallOptions
var passphraseFile, passwordFile, sudoPasswordFile string
var passphraseFile, passwordFile, sudoPasswordFile, tokenFile, workerPasswordFile string
var noActivate bool
fs.StringVar(&opts.Host, "host", "", "SSH server hostname or address")
fs.IntVar(&opts.Port, "port", 22, "SSH server port")
fs.StringVar(&opts.User, "user", "", "SSH username")
@@ -188,8 +193,22 @@ func sourceInstallCommand(args []string) int {
fs.StringVar(&opts.ToolchainDir, "toolchain-dir", "", "remote Go install path ending in /go (default /usr/local/go)")
fs.StringVar(&opts.StageBinary, "stage-binary", "", "staging binary path (default <build-dir>/rsmon-worker)")
fs.DurationVar(&opts.SessionTimeout, "session-timeout", 0, "per-remote-command timeout (default 30m; 0 uses the default)")
fs.BoolVar(&noActivate, "no-activate", false, "stop after the staging build and do not install/start a service (activation is the default)")
fs.StringVar(&opts.Activation.Name, "name", "", "instance name: activates rsmon-worker-<name> with its own config/data/unit/port")
fs.StringVar(&opts.Activation.URL, "url", installer.DefaultURL, "RSMon server URL (RSMON_URL)")
fs.StringVar(&opts.Activation.Token, "token", "", "worker API token (RSMON_TOKEN; required for activation)")
fs.StringVar(&opts.Activation.Token, "api-key", "", "worker API token (alias for --token)")
fs.StringVar(&tokenFile, "token-file", "", "file containing the worker API token")
fs.StringVar(&opts.Activation.PublicURL, "public-url", "", "advertised public origin (PUBLIC_URL; scheme + host, no path)")
fs.StringVar(&opts.Activation.Host, "worker-host", "", "worker webapp bind address (WORKER_HOST; default 127.0.0.1)")
fs.StringVar(&opts.Activation.Port, "worker-port", "", "worker webapp bind port (WORKER_PORT; default 27401, required with --name)")
fs.StringVar(&opts.Activation.Login, "worker-login", "", "operator console login (WORKER_LOGIN; default admin with a generated password)")
fs.StringVar(&opts.Activation.Password, "worker-password", "", "operator console password (WORKER_PASSWORD)")
fs.StringVar(&workerPasswordFile, "worker-password-file", "", "file containing the operator console password")
fs.StringVar(&opts.Activation.EnvFile, "env-file", "", "local systemd-safe env file uploaded for activation (overrides the individual knobs)")
fs.BoolVar(&opts.Activation.NoStart, "no-start", false, "install the binary, env, data dir, and service definition without starting the worker")
fs.Usage = func() {
fmt.Fprintln(fs.Output(), "Usage: rsmon-worker source-install --host HOST --user USER [SSH options] [source options]")
fmt.Fprintln(fs.Output(), "Usage: rsmon-worker source-install --host HOST --user USER [--token TOKEN|--token-file FILE] [SSH options] [source options]")
fs.PrintDefaults()
}
if err := fs.Parse(args); err != nil {
@@ -215,6 +234,15 @@ func sourceInstallCommand(args []string) int {
fmt.Fprintf(os.Stderr, "source-install failed: %v\n", err)
return 1
}
if opts.Activation.Token, err = secretValue(opts.Activation.Token, tokenFile); err != nil {
fmt.Fprintf(os.Stderr, "source-install failed: %v\n", err)
return 1
}
if opts.Activation.Password, err = secretValue(opts.Activation.Password, workerPasswordFile); err != nil {
fmt.Fprintf(os.Stderr, "source-install failed: %v\n", err)
return 1
}
opts.Activation.Activate = !noActivate
res, err := installer.SourceInstall(opts)
if err != nil {
fmt.Fprintf(os.Stderr, "source-install failed: %v\n", err)
@@ -227,6 +255,22 @@ func sourceInstallCommand(args []string) int {
fmt.Printf(" commit: %s\n", res.ResolvedCommit)
fmt.Printf(" record file: %s\n", res.RecordFile)
fmt.Printf(" staged build: %s\n", res.StageBinary)
fmt.Println(" status: not installed as a service (activation is the next milestone)")
if res.Activation != nil {
fmt.Printf(" installed to: %s\n", res.Activation.Binary)
fmt.Printf(" env file: %s (mode 0600)\n", res.Activation.EnvFile)
fmt.Printf(" data dir: %s\n", res.Activation.DataDir)
if res.Activation.UnitFile != "" {
fmt.Printf(" service: %s (%s)\n", res.Activation.UnitFile, res.Activation.UnitName)
} else {
fmt.Println(" service: none (no supported init system detected; supervisor-managed)")
}
if res.Activation.Started {
fmt.Printf(" status: running (supervisor=%s, /healthz verified)\n", res.Activation.Supervisor)
} else {
fmt.Printf(" status: installed (not started, --no-start)\n")
}
} else {
fmt.Println(" status: not installed as a service (--no-activate)")
}
return 0
}

Просмотреть файл

@@ -2,10 +2,7 @@
## Status
In progress. Work package 1 (reusable Docker/OpenSSH harness and distro
fixtures), the pure detection/planning foundation (work package 2 core),
and work package 3 (remote execution through the existing SSH transport)
are implemented:
In progress. Work packages 1-4 are implemented:
- `internal/installer/harness` builds and runs real OpenSSH containers
for Alpine, Ubuntu, and Arch, waits for real network readiness, captures
@@ -24,17 +21,21 @@ are implemented:
downloads and SHA-256-verifies the pinned Go toolchain before
extraction, clones/updates the public repository, checks out the
resolved branch, records the resolved branch and commit, and builds the
worker to a staging path. It deliberately does not install or replace
the running service, configuration, or data directory: atomic
activation and rollback are the next work package.
worker to a staging path.
- Work package 4 (`installer.SourceInstall` with `Activation` enabled)
atomically installs the staged binary, validated environment, data
directory, and the detected init's service definition; starts/restarts
the worker; verifies the process and `/healthz`; and rolls back to the
prior working install on any activation/start/health failure. Reruns
are idempotent with exactly one running worker and no leaked temp
files. The CLI runs the full flow by default and accepts
`--no-activate` (staging only) and `--no-start`.
The current Go installer can still only upload a binary or deploy an
immutable Docker image over SSH. Source installs now build remotely to a
staging path, but service activation over SSH (work package 4) is not
implemented yet; the acceptance test stops after install-to-staging and
idempotent-rerun assertions. Existing tests are unit tests plus the
harness tests that run against live OpenSSH containers when explicitly
enabled.
The current Go installer can also upload a binary or deploy an immutable
Docker image over SSH (`deploy`). Source installs build remotely, then
activate the staged build atomically with rollback. Existing tests are
unit tests plus the harness tests that run against live OpenSSH
containers when explicitly enabled.
## Initial Platform Scope
@@ -66,7 +67,6 @@ It then:
7. atomically installs the binary, validated environment, data directory, and
service definition;
8. starts the service and verifies process status and `/healthz`.
Repository, branch, Go version, checksum source, build directory, and Go module
proxy may be configurable, but production output records their resolved values.
The default repository is publicly readable and requires no source credential.
@@ -79,7 +79,7 @@ Work package 3 is `installer.SourceInstall` in
verification) and runs every remote step with the same privilege path
(root, passwordless sudo, or `sudo -S -p ''` with the password delivered
only over stdin). Steps 1-6 of the flow above are implemented; step 7
(atomic install) is deliberately the next work package.
(atomic install) and step 8 (start + verify) are work package 4.
Per step:
@@ -162,8 +162,89 @@ Security properties of work package 3:
or `/usr/local/.go-staging-*`/`.go-backup` leaks after the rerun.
- Failed builds and failed checkouts leave the previous staging binary
untouched (the binary is only overwritten by an atomically-swapped
successful build, and a checkout failure aborts before the build);
there is no running service or configuration to preserve yet.
successful build, and a checkout failure aborts before the build).
## Work Package 4: Atomic Activation And Rollback
Work package 4 is the second half of `installer.SourceInstall`, driven by
`Activation` (`internal/installer/sourceactivate.go`). It runs over the
same SSH executor, privilege path, and bounded-error machinery as the
staging steps and installs the staged build atomically:
- **Layout.** The classic installer's on-disk layout is reused
(`resolvePaths`): binary `/usr/local/bin/rsmon-worker[-name]`, config
`/etc/rsmon-worker[-name]/worker.env` (mode 0600), data
`/var/lib/rsmon-worker[-name]`, and a service definition for the
detected init (systemd unit `/etc/systemd/system/rsmon-worker.service`
mode 0644, or an OpenRC script `/etc/init.d/rsmon-worker` mode 0755).
A named instance (`--name`) mirrors the classic multi-instance
convention with `-<name>` suffixes and its own unit and port.
- **Environment.** The env file is rendered by the classic installer's
`resolveInstallEnv` + `renderEnvFile` (so `PUBLIC_URL` canonicalization,
basic-auth XOR, required `RSMON_URL`/`RSMON_TOKEN`, and systemd-safe
value validation are identical to `install`). The env file is read
*exactly once* and validated from the in-memory bytes
(`parseEnvironmentContent`), and the rendered env is cached during
option normalization and reused at activation time, so a hostile local
writer cannot swap the file between the preflight and the remote
install (env-file TOCTOU). The rendered bytes are uploaded into a
server-created 0700 temp dir (`mktemp -d /tmp/rsmon-worker-act.XXXXXX`
under the SSH user) rather than a predictable `/tmp` path, eliminating
the symlink/TOCTOU attack surface on the upload while the token still
travels only as base64 over the session stdin and, later, only inside
the mode-0600 env file. The env is then installed with mode 0600 in the
config dir (mode 0750).
- **Atomic install.** The staged binary is validated (`<stage>
--version`) before any state is touched, then installed with a
same-directory temp file + rename. The env and unit files are installed
the same way. The data dir (and `webapp/` subdir) is created with mode
0755.
- **Supervisor.** `restart_svc` drives start/restart through whichever
init system is *actually running*: systemd (`systemctl`), OpenRC
(`rc-service`), or the embedded supervisor fallback when no init is
running (containers/chroots, the "no-service gate"). The health loop
verifies the service is active *through the supervisor* — `systemctl
is-active` for systemd, `rc-service status` for OpenRC, and a
zombie-aware pid check for the embedded supervisor — never through a
pid file on the init-managed paths. The embedded supervisor keeps the
worker as a single background process with a pid file under the data
dir, stops the previous instance before starting, refuses to kill a pid
whose executable is not the configured worker (`/proc/<pid>/exe`
checked before every `kill`), and answers `/healthz` via the worker's
own `liveness` subcommand with a clean `env -i` environment built from
the env file (so a rolled-back env can never leak a stale variable into
the restored process). The OpenRC init script exports the env-file
variables (`set -a` before sourcing) so the worker inherits them when
OpenRC starts it.
- **Rollback.** The prior binary, env, and unit (and the unit's enable
state) are snapshotted into the data dir before any mutation, and an
`EXIT`/`HUP`/`INT`/`TERM` trap restores them (preserving the prior
binary/env/unit metadata via `cp -p`) and restarts the prior service if
any later step fails or a signal interrupts the run. A corrupt staged
binary fails the pre-validation *before* the trap is armed, so the
prior install is never touched. A fresh-install failure disables the
newly-enabled unit (`systemctl disable` / `rc-update del`) and removes
it; a rerun failure restores the prior unit and re-applies its prior
enable state. Rollback also removes the backup dir, the `.new` temp
files, the uploaded env/unit temps, and the activation lock.
- **Lock, recovery, interruption.** A `mkdir`-based activation lock
(`.rsmon-activate.lock`, broken automatically when its recorded pid is
dead) prevents concurrent activations from racing the deployed-state
mutation. The snapshot is marked; a run killed mid-flight (SSH drop,
SIGKILL) leaves that marker, and the next activation restores the
leftover snapshot before proceeding, so the host never stays
half-activated.
- **Idempotency.** A rerun reuses the existing env (rewriting it
deterministically from the same knobs), swaps the binary atomically,
restarts exactly one worker, and leaves no `.rsmon-backup`, `.new`,
`/tmp/rsmon-worker-act.*`, or lock leftovers. `--no-start` installs the
full layout without starting anything and reruns stay idle.
The `source-install` CLI activates by default; pass `--no-activate` for
the staging-only behavior of work package 3. The worker token is required
for activation and is supplied with `--token`/`--token-file` or
`--env-file` (prefer the file options; direct flags expose the value
through the process list).
## Docker OpenSSH Test Harness
@@ -273,22 +354,37 @@ RSMON_TEST_DOCKER=1 go test -v -count=1 -timeout 30m ./internal/installer/harnes
matches, the clone's origin is verified against the configured repository and
then fetched in place, the resolved branch/commit record is rewritten after
the new build succeeds, and the staging build atomically swaps over the
previous staging binary. The "one active service" guarantee is a work package
4 property (activation); work package 3 leaves no running service to
duplicate and no leaked toolchain temp files.
previous staging binary. Activation (work package 4) makes the rerun
idempotent at the service level too: exactly one worker process exists, the
env file is rewritten deterministically from the same knobs, and no
`.rsmon-backup`, `.new`, `/tmp/rsmon-worker-act.*`, or lock files leak.
- Wrong host fingerprints fail before remote mutation. The harness's fresh
known_hosts file is trust-on-first-use; its dedicated mismatch test dials
against a known_hosts entry carrying a different server key and proves the
dial fails before any command runs.
- Tokens/passwords should come from files: `--key-passphrase-file`,
`--password-file`, and `--sudo-password-file` keep secrets out of argv and
shell history, while the equivalent direct flags expose them through the
process list. The source install itself sends no worker token or
control-plane credential at all, and sudo passwords travel only over the
session's stdin.
- Remote temporary files are removed on success and failure.
`--password-file`, `--sudo-password-file`, and the activation
`--token-file`/`--env-file` keep secrets out of argv and shell history,
while the equivalent direct flags expose them through the process list. The
worker token is written only to the mode-0600 env file, never echoed into a
command or log, and sudo passwords travel only over the session's stdin. The
env and unit uploads land in a server-created 0700 `mktemp -d` directory (not
a predictable `/tmp` path), so no local user can plant a symlink at the
upload target (TOCTOU), and the env file itself is read/rendered exactly once
so a local writer cannot swap it between validation and upload.
- Remote temporary files are removed on success and failure, including the
activation backup dir, the uploaded env/unit temps, and the activation lock.
The snapshot is written under a marker so a run killed mid-flight is
recovered (restored) by the next activation instead of leaving the host
half-activated.
- Failed builds, failed checkouts, and failed toolchain swaps do not replace a
working binary or service definition.
working binary or service definition. Failed activations, failed starts, and
failed `/healthz` verifications restore the prior binary, env, and service
definition (preserving their metadata), re-apply the prior unit enable state
(or disable a freshly-enabled unit on a fresh failure), and bring the prior
worker back up (work package 4 rollback). The supervisor only ever kills a
process whose `/proc/<pid>/exe` matches the configured worker binary, so a
stale or recycled pid file can never kill an unrelated process.
- Package-manager and download failures return bounded actionable errors.
- The installer verifies Go tarball checksum before extraction.
@@ -300,24 +396,32 @@ RSMON_TEST_DOCKER=1 go test -v -count=1 -timeout 30m ./internal/installer/harnes
- [x] 3. Execute source installation through the existing SSH transport
(prerequisites, verified Go toolchain, clone/update, resolved branch and
commit record, and a build to a staging path; no service activation).
- [ ] 4. Add atomic build/install, idempotency, and failure rollback.
- [x] 4. Add atomic build/install, idempotency, and failure rollback (binary,
env, data dir, and init service definition installed atomically; process
and `/healthz` verified; activation/start/health failures restore the
prior install; reruns keep exactly one service).
- [ ] 5. Add Alpine, Ubuntu, and Arch network E2E tests to CI.
- [ ] 6. Add CentOS-family support.
- [ ] 7. Plan native Windows service and macOS launchd installers separately.
## Acceptance Gates
- [ ] All three initial Linux images install from a clean state through OpenSSH.
Work package 3 covers the install-to-staging half (package install,
verified Go 1.26, clone/update, resolved commit, staging build); the
service-start half is the work package 4 gate.
- [ ] The built worker reports the expected version/commit and serves `/healthz`.
- [ ] Re-running the installer succeeds without duplicate services or leaked files.
- [ ] Host-key, checksum, clone, build, and service-start failure tests preserve the
previous installation.
- [x] All three initial Linux images install from a clean state through OpenSSH:
package install, verified Go 1.26, clone/update, resolved commit, staging
build, and (work package 4) atomic activation with a running verified
worker.
- [x] The built worker reports the expected version/commit and serves `/healthz`
(verified by the E2E fixture tests and by the activation health gate).
- [x] Re-running the installer succeeds without duplicate services or leaked
files (exactly one worker process and no backup/temp leftovers asserted by
the rerun test on all three fixtures).
- [x] Host-key, checksum, clone, build, and service-start failure tests preserve
the previous installation (build/checkout, activation, start, and health
failure reruns on the Alpine fixture all restore the prior binary, env,
and running service).
- [ ] CI uses approved registry mirrors and cleans every test container/network.
## Verified Test Evidence (work packages 1-3)
## Verified Test Evidence (work packages 1-4)
Recorded 2026-08-12 from `make test-ssh` (Docker Engine 29.7.1):
@@ -334,14 +438,48 @@ Recorded 2026-08-12 from `make test-ssh` (Docker Engine 29.7.1):
prerequisite install, verified Go 1.26 toolchain download/extraction, clone
of the public repository, resolution of its default branch, resolved-commit
record file, and a staging build that reports the resolved commit via
`--version`. The public repo's `HEAD` was `master` at `4651deb2...` during
the run, and all three fixtures resolved to that branch and commit (the
installer records whatever the remote publishes). A rerun succeeds, keeps
the same branch, reuses the toolchain, and leaves no
`/tmp/rsmon-toolchain-*` temp dirs. The running service and its
configuration are not touched (work package 4 boundary).
- The unit suite covers the per-step remote scripts, option validation,
branch/commit parsing, sudo wrapping (password never in the command), the
secrets-absent contract, and an in-process real-SSH orchestration flow with
failure paths for missing pinned branches, build failures, and detection
failures.
`--version`. A rerun succeeds, keeps the same branch, reuses the toolchain,
and leaves no `/tmp/rsmon-toolchain-*` temp dirs.
- `TestSourceInstallActivationFixtures` runs the full work-package-4 flow on
each fixture from a clean state: after the staging build the installer
atomically installs the binary (`/usr/local/bin/rsmon-worker`), env
(`/etc/rsmon-worker/worker.env`, mode 0600; config dir 0750), data dir
(`/var/lib/rsmon-worker`), and the detected init's service definition
(systemd unit on Ubuntu/Arch, OpenRC script on Alpine), then starts the
worker via the embedded supervisor (no init runs inside the fixtures) and
verifies exactly one worker process and a live `/healthz`. A rerun succeeds,
keeps a single process, rewrites the env deterministically, and leaves no
`.rsmon-backup`, `.new`, `/tmp/rsmon-worker-act.*`, or lock leftovers.
- `TestSourceInstallActivationFailureRollback` (Alpine) forces four rerun
failures and proves each restores the prior install byte-for-byte and
running: a dirty checkout (build failure), a sabotaged atomic swap
(activation failure), `WORKER_CLUSTER_ENABLED=true` without credentials (the
new process exits at boot; start failure), and `WORKER_HOST=255.255.255.255`
(the new process runs but `/healthz` is unreachable; health failure). In
every case the installed binary SHA-256, env, service definition, single
process, and liveness match the pre-failure state, and no backup, lock, or
temp files leak.
- `TestSourceInstallActivationNoStart` installs the full layout without
starting anything, and reruns stay idle.
- The targeted shell fixture tests (`TestActivateShell*`) execute the real
activation script against stub `systemctl`/`rc-service`/`rc-update`
implementations and cover the init-managed paths the Docker fixtures cannot
reach: systemd unit install/enable with a `systemctl is-active`-based health
loop (and no pid file), OpenRC install/enable with an `rc-service`-based
health loop, fresh-failure rollback disabling a newly-enabled unit, rerun
rollback restoring the prior unit and its enable state, stale-lock breaking,
and interrupted-run recovery from a leftover backup marker.
- The unit suite covers the per-step remote scripts, the activation script
(backup + recovery marker, activation lock, atomic install, rollback trap,
supervisor, pid/zombie handling, pid-belongs-to-binary checks, enable-state
restoration, secrets absent), OpenRC unit rendering (including env
export), option validation, branch/commit parsing, sudo wrapping (password
never in the command), and an in-process real-SSH orchestration flow with
failure paths for missing pinned branches, build failures, detection
failures, and activation/start/health failures.
- **Residual limitation (stated honestly):** live systemd and OpenRC cannot
run inside the Docker/OpenSSH harness (the container PID 1 is sshd), so the
init-managed supervisor paths are verified by the stub-based shell fixture
tests above rather than against a real init. The stub tools simulate unit
state and command flow, not real systemd/OpenRC unit semantics; a real
init-system smoke test on a booted host remains a follow-up.

Просмотреть файл

@@ -0,0 +1,390 @@
package harness
import (
"bytes"
"context"
"os"
"path/filepath"
"strings"
"testing"
"time"
"golang.org/x/crypto/ssh"
"rocketgit.ru/rsmon/worker/internal/installer"
"rocketgit.ru/rsmon/worker/internal/sshinstall"
)
// testWorkerToken is the operator-supplied worker token used by the
// activation E2E tests. The fixture has no reachable control plane, but
// the worker still binds its webapp and serves /healthz, which is what
// activation verifies.
const testWorkerToken = "e2e-activation-test-token"
// activationOptions builds the work-package-4 source-install options for
// a started fixture: full activation with a fixed test token, honoring
// the RSMON_TEST_SOURCE_REPO/BRANCH overrides the staging tests use.
func activationOptions(h *Harness, f Fixture) installer.SourceInstallOptions {
opts := installer.SourceInstallOptions{
SSHOptions: installer.SSHOptions{
Host: "127.0.0.1",
Port: h.Port(),
User: f.UserOrDefault(),
IdentityFile: testKeyPath(),
KnownHostsFile: h.KnownHostsPath(),
},
Activation: installer.ActivationOptions{
Activate: true,
URL: "https://rsmon.ru",
Token: testWorkerToken,
},
}
if repo := strings.TrimSpace(os.Getenv("RSMON_TEST_SOURCE_REPO")); repo != "" {
opts.Repo = repo
}
if branch := strings.TrimSpace(os.Getenv("RSMON_TEST_SOURCE_BRANCH")); branch != "" {
opts.Branch = branch
}
return opts
}
// fixtureByName returns the fixture entry for a fixture name.
func fixtureByName(t *testing.T, name string) Fixture {
t.Helper()
for _, f := range Fixtures() {
if f.Name == name {
return f
}
}
t.Fatalf("no fixture named %q", name)
return Fixture{}
}
// startActivatedFixture starts a fixture container and runs a full
// source install with activation against it, returning the harness, the
// options (so a test can rerun with modifications), the result, and a
// live SSH client. The caller owns client.Close and h.Stop (registered as
// a test cleanup).
func startActivatedFixture(t *testing.T, name string) (*Harness, installer.SourceInstallOptions, *installer.SourceInstallResult, *ssh.Client) {
t.Helper()
f := fixtureByName(t, name)
h, err := New("activate-"+name, f)
if err != nil {
t.Fatal(err)
}
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute)
defer cancel()
if err := h.Start(ctx); err != nil {
t.Fatalf("start %s fixture: %v", name, err)
}
t.Cleanup(func() {
if err := h.Stop(); err != nil {
t.Errorf("stop %s fixture: %v", name, err)
}
})
opts := activationOptions(h, f)
res, err := installer.SourceInstall(opts)
if err != nil {
t.Fatalf("activated source install on %s: %v", name, err)
}
client, err := h.Dial()
if err != nil {
t.Fatalf("dial %s after install: %v", name, err)
}
return h, opts, res, client
}
// countWorkerProcesses returns the number of running processes whose
// command line is exactly the installed worker binary. The anchored -f
// pattern matches both busybox and procps pgrep (busybox's -x matches the
// full argv, so -x cannot be used portably).
func countWorkerProcesses(t *testing.T, client *ssh.Client, binary string) string {
t.Helper()
out, err := RunCommand(client, "pgrep -f '^"+binary+"$' | wc -l")
if err != nil {
t.Fatalf("count worker processes: %v", err)
}
return strings.TrimSpace(string(out))
}
// assertActivatedService verifies the full work-package-4 remote state:
// the installed layout and permissions, the native service definition for
// the detected init, exactly one running worker process, the pid file,
// and a live /healthz (via the installed binary's own liveness
// subcommand).
func assertActivatedService(t *testing.T, client *ssh.Client, name string, res *installer.SourceInstallResult) {
t.Helper()
a := res.Activation
if a == nil {
t.Fatalf("%s: activation result missing", name)
}
out, err := RunCommand(client, "test -x "+shellQuote(a.Binary)+" && echo BIN_OK")
if err != nil || !strings.Contains(string(out), "BIN_OK") {
t.Fatalf("%s: installed binary not present/executable at %s: %q, %v", name, a.Binary, out, err)
}
envMode, err := RunCommand(client, "stat -c '%a' "+shellQuote(a.EnvFile))
if err != nil || strings.TrimSpace(string(envMode)) != "600" {
t.Fatalf("%s: env file mode = %q, want 600 (%v)", name, envMode, err)
}
cfgMode, err := RunCommand(client, "stat -c '%a' "+shellQuote(a.ConfigDir))
if err != nil || strings.TrimSpace(string(cfgMode)) != "750" {
t.Fatalf("%s: config dir mode = %q, want 750 (%v)", name, cfgMode, err)
}
if _, err := RunCommand(client, "test -d "+shellQuote(a.DataDir)+"/webapp && echo DATA_OK"); err != nil {
t.Fatalf("%s: data dir not created at %s: %v", name, a.DataDir, err)
}
// The fixtures run no init system (sshd is PID 1), so activation
// must use the embedded supervisor; the native service definition is
// still installed for real hosts.
if a.Supervisor != "none" {
t.Fatalf("%s: supervisor = %q, want none in the fixture", name, a.Supervisor)
}
unitMode := "644"
if res.Detection.InitSystem == sshinstall.InitOpenRC {
unitMode = "755"
}
if a.UnitFile == "" {
t.Fatalf("%s: no service definition installed", name)
}
if _, err := RunCommand(client, "test -f "+shellQuote(a.UnitFile)+" && echo UNIT_OK"); err != nil {
t.Fatalf("%s: unit file missing at %s: %v", name, a.UnitFile, err)
}
gotUnitMode, err := RunCommand(client, "stat -c '%a' "+shellQuote(a.UnitFile))
if err != nil || strings.TrimSpace(string(gotUnitMode)) != unitMode {
t.Fatalf("%s: unit mode = %q, want %s (%v)", name, gotUnitMode, unitMode, err)
}
// Exactly one worker process and a live /healthz.
if got := countWorkerProcesses(t, client, a.Binary); got != "1" {
t.Fatalf("%s: worker process count = %q, want 1", name, got)
}
if out, err := RunCommand(client, shellQuote(a.Binary)+" liveness"); err != nil || !strings.Contains(string(out), "ok") {
t.Fatalf("%s: installed worker /healthz check failed: %q, %v", name, out, err)
}
pidOut, err := RunCommand(client, "cat "+shellQuote(a.DataDir)+"/worker.pid")
if err != nil {
t.Fatalf("%s: pid file missing: %v", name, err)
}
if _, err := RunCommand(client, "kill -0 "+strings.TrimSpace(string(pidOut))+" && echo PID_OK"); err != nil {
t.Fatalf("%s: pid %s not alive: %v", name, pidOut, err)
}
}
// assertNoActivationLeaks verifies a rerun left no backup dir, no lock,
// no /tmp upload leftovers, and no extra worker processes.
func assertNoActivationLeaks(t *testing.T, client *ssh.Client, name string, dataDir string) {
t.Helper()
out, err := RunCommand(client,
"leak=0; [ -e "+shellQuote(dataDir)+"/.rsmon-backup ] && leak=1; "+
"[ -d "+shellQuote(dataDir)+"/.rsmon-activate.lock ] && leak=1; "+
"ls /tmp 2>/dev/null | grep -q '^rsmon-worker-act\\.' && leak=1; "+
"[ \"$leak\" -eq 1 ] && echo LEAK || echo CLEAN")
if err != nil || strings.TrimSpace(string(out)) != "CLEAN" {
t.Fatalf("%s: activation leftovers after rerun: %q, %v", name, out, err)
}
}
// installedBinarySHA captures the SHA-256 of the installed worker binary
// so failure tests can prove the prior binary is preserved.
func installedBinarySHA(t *testing.T, client *ssh.Client, a *installer.ActivationResult) []byte {
t.Helper()
out, err := RunCommand(client, "sha256sum "+shellQuote(a.Binary))
if err != nil {
t.Fatal(err)
}
return bytes.TrimSpace(out)
}
// TestSourceInstallActivationFixtures is the work-package-4 success and
// idempotency acceptance test: on every fixture a full source install
// ends with an atomically installed binary, a mode-0600 env file, the
// data dir, a service definition for the detected init, and a running
// verified worker. A rerun succeeds, keeps a single worker process,
// rewrites the env deterministically, and leaks no temp or backup files.
//
// Opt-in: set RSMON_TEST_DOCKER=1 (see make test-ssh).
func TestSourceInstallActivationFixtures(t *testing.T) {
SkipUnlessEnabled(t)
for _, f := range Fixtures() {
f := f
t.Run(f.Name, func(t *testing.T) {
_, opts, res, client := startActivatedFixture(t, f.Name)
defer client.Close() //nolint:errcheck
assertActivatedService(t, client, f.Name, res)
// Rerun: idempotent, single service, no leaks.
res2, err := installer.SourceInstall(opts)
if err != nil {
t.Fatalf("%s: activated rerun: %v", f.Name, err)
}
if res2.Activation == nil || res2.ResolvedCommit != res.ResolvedCommit {
t.Fatalf("%s: rerun lost activation/commit: %+v", f.Name, res2)
}
assertActivatedService(t, client, f.Name, res2)
assertNoActivationLeaks(t, client, f.Name, res2.Activation.DataDir)
envAfter, err := RunCommand(client, "cat "+shellQuote(res2.Activation.EnvFile))
if err != nil {
t.Fatalf("%s: read env after rerun: %v", f.Name, err)
}
if !strings.Contains(string(envAfter), "RSMON_TOKEN="+testWorkerToken) {
t.Fatalf("%s: env lost the worker token after rerun: %q", f.Name, envAfter)
}
})
}
}
// TestSourceInstallActivationFailureRollback is the work-package-4
// rollback acceptance test. After a successful activated install on the
// Alpine fixture, each rerun is forced to fail at a different stage -
// build (dirty checkout), activation (atomic swap sabotaged), start (the
// worker process dies at boot), and health (/healthz unreachable) - and
// every failure must leave the previous working install running with the
// same binary, the same single process, and /healthz answering.
//
// Opt-in: set RSMON_TEST_DOCKER=1 (see make test-ssh).
func TestSourceInstallActivationFailureRollback(t *testing.T) {
SkipUnlessEnabled(t)
const fixture = "alpine"
_, opts, res, client := startActivatedFixture(t, fixture)
defer client.Close() //nolint:errcheck
a := res.Activation
assertActivatedService(t, client, fixture, res)
// 1. Build failure preserves the prior install.
// A dirty tracked tree makes the rerun's checkout refuse before
// the build, so neither staging nor the service changes.
if _, err := RunCommand(client, "git -C "+shellQuote(res.Plan.BuildDir)+" checkout -q master~1 -- Makefile"); err != nil {
t.Fatalf("dirty the working tree: %v", err)
}
before := installedBinarySHA(t, client, a)
_, err := installer.SourceInstall(opts)
if err == nil || !strings.Contains(err.Error(), "check out branch") {
t.Fatalf("rerun on dirty tree err = %v, want checkout failure", err)
}
assertActivatedService(t, client, fixture, res)
if !bytes.Equal(before, installedBinarySHA(t, client, a)) {
t.Fatalf("installed binary changed after build-failure rerun")
}
// Restore the tree so later reruns can check out again.
if _, err := RunCommand(client, "git -C "+shellQuote(res.Plan.BuildDir)+" checkout -q HEAD -- Makefile"); err != nil {
t.Fatalf("restore working tree: %v", err)
}
// 2. Activation failure preserves the prior install.
// Pre-creating the atomic-swap temp path as a directory makes the
// binary install step fail mid-activation; the rollback trap must
// restore the previous binary/env and restart the prior service.
if _, err := RunCommand(client, "mkdir -p /usr/local/bin/rsmon-worker.new"); err != nil {
t.Fatalf("sabotage the atomic binary swap: %v", err)
}
_, err = installer.SourceInstall(opts)
if err == nil || !strings.Contains(err.Error(), "activate worker service") {
t.Fatalf("activation-failure rerun err = %v, want activate failure", err)
}
assertActivatedService(t, client, fixture, res)
if !bytes.Equal(before, installedBinarySHA(t, client, a)) {
t.Fatalf("installed binary changed after activation-failure rerun")
}
assertNoActivationLeaks(t, client, fixture, a.DataDir)
if _, err := RunCommand(client, "rm -rf /usr/local/bin/rsmon-worker.new"); err != nil {
t.Fatalf("clean the swap sabotage: %v", err)
}
// 3. Start failure preserves the prior install.
// WORKER_CLUSTER_ENABLED=true without WORKER_LOGIN/PASSWORD makes
// the new worker exit at boot (cluster init is fatal), so the
// activation supervisor detects the process died and rolls back.
envPath := filepath.Join(t.TempDir(), "worker.env")
startEnv := "RSMON_URL=https://rsmon.ru\nRSMON_TOKEN=" + testWorkerToken + "\nWORKER_CLUSTER_ENABLED=true\n"
if err := os.WriteFile(envPath, []byte(startEnv), 0o600); err != nil {
t.Fatal(err)
}
startOpts := opts
startOpts.Activation.EnvFile = envPath
_, err = installer.SourceInstall(startOpts)
if err == nil || !strings.Contains(err.Error(), "start failure") {
t.Fatalf("start-failure rerun err = %v, want start failure", err)
}
assertActivatedService(t, client, fixture, res)
if !bytes.Equal(before, installedBinarySHA(t, client, a)) {
t.Fatalf("installed binary changed after start-failure rerun")
}
assertNoActivationLeaks(t, client, fixture, a.DataDir)
// 4. Health failure preserves the prior install.
// WORKER_HOST=255.255.255.255 makes the new worker bind fail and
// /healthz unreachable; the process stays up but the health gate
// fails and the rollback restores the prior loopback deployment.
healthOpts := opts
healthOpts.Activation.Host = "255.255.255.255"
_, err = installer.SourceInstall(healthOpts)
if err == nil || !strings.Contains(err.Error(), "health failure") {
t.Fatalf("health-failure rerun err = %v, want health failure", err)
}
assertActivatedService(t, client, fixture, res)
if !bytes.Equal(before, installedBinarySHA(t, client, a)) {
t.Fatalf("installed binary changed after health-failure rerun")
}
assertNoActivationLeaks(t, client, fixture, a.DataDir)
}
// TestSourceInstallActivationNoStart installs the full layout (binary,
// env, data dir, service definition) without starting the worker, and
// asserts nothing runs afterward. Reruns stay idempotent.
//
// Opt-in: set RSMON_TEST_DOCKER=1 (see make test-ssh).
func TestSourceInstallActivationNoStart(t *testing.T) {
SkipUnlessEnabled(t)
const fixture = "alpine"
f := fixtureByName(t, fixture)
h, err := New("nostart-"+fixture, f)
if err != nil {
t.Fatal(err)
}
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute)
defer cancel()
if err := h.Start(ctx); err != nil {
t.Fatalf("start %s fixture: %v", fixture, err)
}
t.Cleanup(func() {
if err := h.Stop(); err != nil {
t.Errorf("stop: %v", err)
}
})
opts := activationOptions(h, f)
opts.Activation.NoStart = true
res, err := installer.SourceInstall(opts)
if err != nil {
t.Fatalf("no-start install: %v", err)
}
client, err := h.Dial()
if err != nil {
t.Fatalf("dial: %v", err)
}
defer client.Close() //nolint:errcheck
a := res.Activation
if a == nil || a.Started {
t.Fatalf("no-start activation result = %+v", a)
}
if out, err := RunCommand(client, "test -x "+shellQuote(a.Binary)+" && test -f "+shellQuote(a.EnvFile)+" && echo INSTALLED"); err != nil || !strings.Contains(string(out), "INSTALLED") {
t.Fatalf("no-start did not install the layout: %q, %v", out, err)
}
if got := countWorkerProcesses(t, client, a.Binary); got != "0" {
t.Fatalf("no-start left a running worker: %q", got)
}
// A rerun is idempotent and still starts nothing.
if _, err := installer.SourceInstall(opts); err != nil {
t.Fatalf("no-start rerun: %v", err)
}
if got := countWorkerProcesses(t, client, a.Binary); got != "0" {
t.Fatalf("no-start rerun left a running worker: %q", got)
}
assertNoActivationLeaks(t, client, fixture, a.DataDir)
}

Просмотреть файл

@@ -325,6 +325,9 @@ func TestSourceInstallFixtures(t *testing.T) {
IdentityFile: testKeyPath(),
KnownHostsFile: h.KnownHostsPath(),
},
// Staging-only: service activation (work package 4) is
// exercised by the dedicated activation tests.
Activation: installer.ActivationOptions{Activate: false},
}
if repo := strings.TrimSpace(os.Getenv("RSMON_TEST_SOURCE_REPO")); repo != "" {
opts.Repo = repo
@@ -446,6 +449,7 @@ func TestSourceInstallDirtyCheckoutPreservesStaging(t *testing.T) {
IdentityFile: testKeyPath(),
KnownHostsFile: h.KnownHostsPath(),
},
Activation: installer.ActivationOptions{Activate: false},
}
if repo := strings.TrimSpace(os.Getenv("RSMON_TEST_SOURCE_REPO")); repo != "" {
opts.Repo = repo

Просмотреть файл

@@ -478,6 +478,16 @@ func ValidateEnvironmentFile(path string) error {
if err != nil {
return fmt.Errorf("read worker environment: %w", err)
}
_, err = parseEnvironmentContent(data)
return err
}
// parseEnvironmentContent validates systemd-safe KEY=VALUE environment
// content and returns the parsed map. Values must not use quoting,
// interpolation, or whitespace; RSMON_URL and RSMON_TOKEN are required and
// must not be duplicated. Operating on the already-read bytes (rather
// than a path) keeps callers free of read-to-validate TOCTOU races.
func parseEnvironmentContent(data []byte) (map[string]string, error) {
values := make(map[string]string)
seenRequired := make(map[string]bool)
for number, line := range strings.Split(string(data), "\n") {
@@ -486,30 +496,30 @@ func ValidateEnvironmentFile(path string) error {
continue
}
if strings.ContainsRune(line, '\r') {
return fmt.Errorf("worker environment line %d contains a carriage return", lineNumber)
return nil, fmt.Errorf("worker environment line %d contains a carriage return", lineNumber)
}
key, value, ok := strings.Cut(line, "=")
if !ok || !envKeyPattern.MatchString(key) {
return fmt.Errorf("worker environment line %d must use KEY=VALUE syntax", lineNumber)
return nil, fmt.Errorf("worker environment line %d must use KEY=VALUE syntax", lineNumber)
}
if err := validateEnvValue(key, value); err != nil {
return fmt.Errorf("worker environment line %d: %w", lineNumber, err)
return nil, fmt.Errorf("worker environment line %d: %w", lineNumber, err)
}
if key == "RSMON_URL" || key == "RSMON_TOKEN" {
if seenRequired[key] {
return fmt.Errorf("worker environment line %d duplicates %s", lineNumber, key)
return nil, fmt.Errorf("worker environment line %d duplicates %s", lineNumber, key)
}
seenRequired[key] = true
}
values[key] = value
}
if err := ValidateURL(values["RSMON_URL"]); err != nil {
return err
return nil, err
}
if err := ValidateToken(values["RSMON_TOKEN"]); err != nil {
return err
return nil, err
}
return nil
return values, nil
}
// validateEnvValue enforces the value rules shared by the strict env

769
internal/installer/sourceactivate.go Обычный файл
Просмотреть файл

@@ -0,0 +1,769 @@
package installer
import (
"errors"
"fmt"
"os"
"strings"
"rocketgit.ru/rsmon/worker/internal/sshinstall"
)
// ActivationOptions drives work package 4 of docs/source-installation.md:
// the atomic activation of the staged worker binary. It reuses the
// classic installer's environment resolution (resolveInstallEnv /
// renderEnvFile), on-disk layout (resolvePaths), and hardened systemd
// unit renderer, then installs the binary, env file, data dir, and the
// detected init's service definition atomically, starts/restarts the
// worker, and verifies the process and /healthz before declaring the
// activation a success. Any activation/start/health failure rolls back to
// the previous working install.
type ActivationOptions struct {
// Activate performs the atomic install + service activation after the
// staging build. When false SourceInstall stops at the staging build
// (the work-package-3 boundary) and touches no service configuration.
Activate bool
// Name is the instance name ("" = the primary rsmon-worker service).
// A named instance gets rsmon-worker-<name> paths and its own unit,
// matching the classic installer's multi-instance layout.
Name string
// URL is RSMON_URL (default https://rsmon.ru).
URL string
// Token is the worker API token (RSMON_TOKEN). Required for
// activation; prefer supplying it through a file at the CLI layer so
// it never appears in the process list.
Token string
// PublicURL is the advertised public origin (PUBLIC_URL).
PublicURL string
// Host is the worker webapp bind address (WORKER_HOST; default
// 127.0.0.1 as resolved by the classic installer).
Host string
// Port is the worker webapp bind port (WORKER_PORT; default 27401,
// required for named instances).
Port string
// Login / Password are the operator-console basic-auth credentials
// (WORKER_LOGIN / WORKER_PASSWORD). Both must be set or both empty.
Login string
Password string
// EnvFile is a local systemd-safe env file uploaded instead of the
// individual knobs. Validated with the classic strict parser.
EnvFile string
// NoStart installs the binary, env, data dir, and service definition
// without starting or restarting the worker.
NoStart bool
}
// ActivationResult records where and how the staged worker was activated.
type ActivationResult struct {
Binary string // installed binary path
ConfigDir string
EnvFile string
DataDir string
UnitFile string // empty when no service definition was installed
UnitName string // systemd unit / rc-service name
Supervisor string // "systemd", "openrc", or "supervisor" (embedded fallback)
Started bool
}
// installOptions projects the activation knobs onto the classic
// installer's option shape so the shared env resolution is reused.
func (a ActivationOptions) installOptions() InstallOptions {
return InstallOptions{
URL: a.URL,
Token: a.Token,
PublicURL: a.PublicURL,
Host: a.Host,
Port: a.Port,
Login: a.Login,
Password: a.Password,
Name: strings.TrimSpace(a.Name),
NoStart: a.NoStart,
EnvFile: a.EnvFile,
}
}
// renderEnv resolves the full worker environment exactly as the classic
// installer would and renders the canonical systemd-safe env file. The
// env file (when configured) is read exactly once and validated from the
// in-memory bytes, so a hostile local writer cannot swap the file
// between the read used for validation and the read used for the render.
func (a ActivationOptions) renderEnv() ([]byte, error) {
name := strings.TrimSpace(a.Name)
if err := validateInstanceName(name); err != nil {
return nil, err
}
var fileEnv map[string]string
if a.EnvFile != "" {
data, err := os.ReadFile(a.EnvFile)
if err != nil {
return nil, fmt.Errorf("--env-file: %w", err)
}
fe, perr := parseEnvironmentContent(data)
if perr != nil {
return nil, fmt.Errorf("--env-file: %w", perr)
}
fileEnv = fe
}
values, err := resolveInstallEnv(a.installOptions(), name, fileEnv)
if err != nil {
return nil, err
}
return renderEnvFile(values), nil
}
// unitContent renders the service definition for the detected init
// system. systemd gets the classic hardened unit; OpenRC gets a native
// init script. An unknown init produces no unit (the no-service gate:
// the embedded supervisor still installs and manages the worker, but no
// native service definition is written).
func unitContent(init sshinstall.InitSystem, p paths) (content string, file string, mode string) {
svcName := strings.TrimSuffix(p.unitName, ".service")
switch init {
case sshinstall.InitSystemd:
return systemdUnitFor(p), p.unitFile, "0644"
case sshinstall.InitOpenRC:
return openrcInitFor(p), "/etc/init.d/" + svcName, "0755"
default:
return "", "", ""
}
}
// runningInitScript reports the init system that is *actually running*
// (not merely installed): systemd, openrc, or none. The installer writes
// the service definition for the detected init but drives start/restart
// through whichever supervisor is usable right now; a container or chroot
// where no init runs falls back to the embedded supervisor.
const runningInitScript = `set -eu
if [ -d /run/systemd/system ] && command -v systemctl >/dev/null 2>&1; then
echo systemd
elif [ -e /run/openrc/softlevel ] && command -v rc-service >/dev/null 2>&1; then
echo openrc
else
echo none
fi`
// normalizeSupervisor bounds the running-init probe output to the three
// supervisor kinds the activation script understands.
func normalizeSupervisor(raw string) string {
switch strings.TrimSpace(raw) {
case "systemd", "openrc":
return strings.TrimSpace(raw)
default:
return "none"
}
}
// activateScript renders the single idempotent remote activation script.
// It is structured as a transaction over the previous install:
//
// 1. acquire an activation lock so concurrent runs cannot mutate the same
// deployment, and recover any interrupted prior activation from its
// leftover backup marker;
// 2. validate the staged binary (fails before any state is touched);
// 3. snapshot the prior binary/env/unit (and the unit's enable state)
// into a backup dir and write a recovery marker;
// 4. arm a rollback EXIT/HUP/INT/TERM trap that restores the snapshot,
// restores the prior enable state (or disables a freshly-installed
// unit on a fresh failure), and restarts the prior service;
// 5. install the new binary, env (mode 0600), data dir, and service
// definition atomically (temp file + rename);
// 6. start/restart through the active supervisor and verify the service
// stays active (systemd `is-active` / rc-service `status` / the
// embedded supervisor's zombie-aware pid check) and answers /healthz;
// 7. on success, drop the backup and release the lock; on failure the
// trap restores the prior install and the script exits non-zero.
//
// Reruns are idempotent: restart always stops the previous instance
// first, so exactly one worker process exists, and every temp file
// (backup dir, .new files, uploaded env/unit, lock) is removed on success
// and failure.
func activateScript(p activateParams) string {
r := strings.NewReplacer(
"@@STAGE@@", shellQuote(p.Stage),
"@@BINARY@@", shellQuote(p.Binary),
"@@CONFIG_DIR@@", shellQuote(p.ConfigDir),
"@@ENV_FILE@@", shellQuote(p.EnvFile),
"@@ENV_TMP@@", shellQuote(p.EnvTmp),
"@@DATA_DIR@@", shellQuote(p.DataDir),
"@@UNIT_TMP@@", shellQuote(p.UnitTmp),
"@@UNIT_FILE@@", shellQuote(p.UnitFile),
"@@UNIT_MODE@@", p.UnitMode,
"@@UNIT_NAME@@", shellQuote(p.UnitName),
"@@RC_NAME@@", shellQuote(p.RCName),
"@@SUPERVISOR@@", p.Supervisor,
"@@NO_START@@", p.NoStart,
)
return r.Replace(activateTemplate)
}
// activateTemplate is the remote script body. Sentinels (@@..@@) are
// substituted by activateScript; every operator-controlled value is
// single-quoted and validated on the controller side first.
const activateTemplate = `set -eu
stage=@@STAGE@@
binary=@@BINARY@@
config_dir=@@CONFIG_DIR@@
env_file=@@ENV_FILE@@
env_tmp=@@ENV_TMP@@
data_dir=@@DATA_DIR@@
unit_tmp=@@UNIT_TMP@@
unit_file=@@UNIT_FILE@@
unit_mode=@@UNIT_MODE@@
unit_name=@@UNIT_NAME@@
rc_name=@@RC_NAME@@
supervisor=@@SUPERVISOR@@
no_start=@@NO_START@@
backup_dir="$data_dir/.rsmon-backup"
lock_dir="$data_dir/.rsmon-activate.lock"
pid_file="$data_dir/worker.pid"
log_file="$data_dir/worker.log"
run_env() {
# Execute "$@" with a fresh environment built only from the env file
# plus PATH, HOME, and the webapp data dir. Sourcing the env file in a
# clean env -i shell prevents variables from a previous activation
# (for example WORKER_CLUSTER_ENABLED) from leaking into the new
# worker process after a rollback restores an older env file.
env -i PATH="/usr/bin:/bin:/sbin:/usr/sbin" HOME="$data_dir" \
RSMON_WEBAPP_DATA_DIR="$data_dir/webapp" \
RSMON_WORKER_ENV_FILE="$env_file" sh -c '
set -a
. "$RSMON_WORKER_ENV_FILE"
set +a
unset RSMON_WORKER_ENV_FILE
exec "$@"
' sh "$@"
}
process_up() {
# True when the given pid is a live, non-zombie process. A zombie
# still answers kill -0 (its task struct exists until reaped), and in
# a container where PID 1 does not reap children a killed worker can
# linger as a zombie for a long time, so the /proc state must be
# checked explicitly.
[ -n "$1" ] || return 1
kill -0 "$1" 2>/dev/null || return 1
state="$(awk '{print $3}' "/proc/$1/stat" 2>/dev/null || true)"
[ "$state" != "Z" ] || return 1
}
process_is_worker() {
# True when the given pid's executable is (or was, before an atomic
# binary swap) the configured worker binary, so a stale or recycled
# pid file can never make the supervisor kill an unrelated process.
[ -n "$1" ] || return 1
exe="$(readlink "/proc/$1/exe" 2>/dev/null || true)"
case "$exe" in
"$binary"|"$binary (deleted)") return 0 ;;
esac
return 1
}
stop_process() {
if [ -f "$pid_file" ]; then
pid="$(cat "$pid_file" 2>/dev/null || true)"
if [ -n "$pid" ] && process_is_worker "$pid" && process_up "$pid"; then
kill "$pid" 2>/dev/null || true
i=0
while [ "$i" -lt 10 ]; do
if ! process_up "$pid"; then
break
fi
sleep 1
i=$((i + 1))
done
kill -9 "$pid" 2>/dev/null || true
fi
fi
rm -f "$pid_file"
}
start_process() {
stop_process
mkdir -p "$data_dir"
run_env sh -c 'nohup "$1" >>"$2" 2>&1 & echo $! > "$3"' sh "$binary" "$log_file" "$pid_file"
}
process_alive() {
[ -f "$pid_file" ] || return 1
pid="$(cat "$pid_file" 2>/dev/null || true)"
process_is_worker "$pid" || return 1
process_up "$pid"
}
worker_come_up() {
# Wait up to 10s for the freshly-started worker to become a live,
# non-zombie process running the configured binary. The recorded pid
# starts as the nohup/sh child before it execs the worker, so early
# readlink checks can transiently see the interpreter; this retry both
# tolerates that exec window and catches genuine immediate deaths.
i=0
while [ "$i" -lt 10 ]; do
if process_alive; then
return 0
fi
sleep 1
i=$((i + 1))
done
return 1
}
health_ok() {
run_env "$binary" liveness >/dev/null 2>&1
}
svc_active() {
# The health loop verifies the service is active through the
# supervisor (never through a pid file for the init-managed paths).
case "$supervisor" in
systemd) systemctl is-active --quiet "$unit_name" ;;
openrc) rc-service "$rc_name" status >/dev/null 2>&1 ;;
none) process_alive ;;
esac
}
stop_svc() {
case "$supervisor" in
systemd) systemctl stop "$unit_name" >/dev/null 2>&1 || true ;;
openrc) rc-service "$rc_name" stop >/dev/null 2>&1 || true ;;
none) stop_process ;;
esac
}
restart_svc() {
case "$supervisor" in
systemd)
systemctl daemon-reload >/dev/null 2>&1
if ! systemctl restart "$unit_name" >/dev/null 2>&1; then
echo "start failure: systemctl restart $unit_name failed" >&2
return 1
fi
if ! systemctl is-active --quiet "$unit_name"; then
echo "start failure: unit $unit_name is not active" >&2
return 1
fi
;;
openrc)
if ! rc-service "$rc_name" restart >/dev/null 2>&1; then
echo "start failure: rc-service restart $rc_name failed" >&2
return 1
fi
if ! rc-service "$rc_name" status >/dev/null 2>&1; then
echo "start failure: service $rc_name is not running" >&2
return 1
fi
;;
none)
start_process
if ! worker_come_up; then
echo "start failure: worker process did not stay alive" >&2
return 1
fi
;;
esac
i=0
while [ "$i" -lt 30 ]; do
if health_ok; then
return 0
fi
# A worker that stops while the health gate is still probing is a
# start failure (it never became a stable service), not merely an
# unhealthy-but-running one.
if ! svc_active; then
echo "start failure: worker service is not active" >&2
return 1
fi
sleep 1
i=$((i + 1))
done
echo "health failure: worker did not answer /healthz" >&2
return 1
}
# --- unit enable helpers -------------------------------------------------
# unit_was_enabled records whether the unit referenced by the current
# params is enabled (best-effort; a missing init leaves it disabled).
unit_was_enabled() {
case "$supervisor" in
systemd) systemctl is-enabled "$unit_name" >/dev/null 2>&1 && unit_enabled=1 ;;
openrc) [ -e "/etc/runlevels/default/$rc_name" ] && unit_enabled=1 ;;
esac
return 0
}
# set_unit_enabled "$1" enables (1) or disables (0) the unit.
set_unit_enabled() {
case "$supervisor" in
systemd)
systemctl daemon-reload >/dev/null 2>&1 || true
if [ "$1" -eq 1 ]; then
systemctl enable "$unit_name" >/dev/null 2>&1 || true
else
systemctl disable "$unit_name" >/dev/null 2>&1 || true
fi
;;
openrc)
if [ "$1" -eq 1 ]; then
rc-update add "$rc_name" default >/dev/null 2>&1 || true
else
rc-update del "$rc_name" default >/dev/null 2>&1 || true
fi
;;
esac
return 0
}
# --- activation lock ------------------------------------------------------
# Prevent concurrent activations on the same host from racing the
# deployed-state mutation. A stale lock (left over from a SIGKILL'd run
# whose pid is no longer alive) is broken automatically.
acquire_lock() {
mkdir -p "$data_dir"
if [ -d "$lock_dir" ]; then
lockpid="$(cat "$lock_dir/pid" 2>/dev/null || true)"
if [ -n "$lockpid" ] && ! kill -0 "$lockpid" 2>/dev/null; then
rm -rf "$lock_dir" 2>/dev/null || true
fi
fi
i=0
while [ "$i" -lt 60 ]; do
if mkdir "$lock_dir" 2>/dev/null; then
chmod 0700 "$lock_dir" 2>/dev/null || true
echo $$ > "$lock_dir/pid" 2>/dev/null || true
return 0
fi
sleep 1
i=$((i + 1))
done
echo "another rsmon-worker activation is in progress ($lock_dir)" >&2
return 1
}
release_lock() {
rm -rf "$lock_dir" 2>/dev/null || true
}
if ! acquire_lock; then
exit 1
fi
# Any early failure (before the rollback trap is armed) still releases the
# lock.
trap 'release_lock' EXIT
# --- interrupted-run recovery --------------------------------------------
# A previous activation that was killed mid-flight (SSH drop, SIGKILL)
# leaves its backup marker behind. Restore that snapshot before the fresh
# activation runs, so the host never stays half-activated and the next run
# starts from a consistent prior state.
unit_enabled=0
recover_backup() {
if [ ! -f "$backup_dir/.marker" ]; then
return 0
fi
echo "recovering interrupted activation from $backup_dir" >&2
if [ -f "$backup_dir/binary" ]; then
mkdir -p "$(dirname "$binary")" || true
cp -p "$backup_dir/binary" "$binary" || true
fi
if [ -f "$backup_dir/worker.env" ]; then
mkdir -p "$config_dir" || true
cp -p "$backup_dir/worker.env" "$env_file" || true
fi
if [ -n "$unit_file" ] && [ -f "$backup_dir/unit" ]; then
mkdir -p "$(dirname "$unit_file")" || true
cp -p "$backup_dir/unit" "$unit_file" || true
chmod "$unit_mode" "$unit_file" || true
fi
if [ "$(cat "$backup_dir/.marker" 2>/dev/null || true)" = "1" ]; then
unit_enabled=1
fi
set_unit_enabled "$unit_enabled" || true
if [ -f "$backup_dir/binary" ] && [ -f "$backup_dir/worker.env" ] && [ "$no_start" -ne 1 ]; then
restart_svc || true
fi
rm -rf "$backup_dir" || true
}
recover_backup
# --- staged binary validation (no state touched) -------------------------
# A corrupt or missing staging build aborts here and leaves the prior
# install completely untouched (the rollback trap is not armed yet).
"$stage" --version >/dev/null
# --- snapshot the prior install ------------------------------------------
rm -rf "$backup_dir"
mkdir -p "$backup_dir"
had_binary=0
had_env=0
had_unit=0
unit_enabled=0
if [ -e "$binary" ]; then
cp -p "$binary" "$backup_dir/binary"
had_binary=1
fi
if [ -e "$env_file" ]; then
cp -p "$env_file" "$backup_dir/worker.env"
had_env=1
fi
if [ -n "$unit_file" ] && [ -e "$unit_file" ]; then
had_unit=1
cp -p "$unit_file" "$backup_dir/unit"
unit_was_enabled
fi
# The marker is written only after the snapshot copies so recovery never
# sees a partial snapshot; it records whether the prior unit was enabled.
printf '%s\n' "$unit_enabled" > "$backup_dir/.marker"
# --- rollback trap --------------------------------------------------------
# Any failure from here on restores the snapshot (preserving binary/env/unit
# metadata), restores the prior enable state (or disables a freshly-installed
# unit on a fresh install), and brings the prior service back up. The trap
# also fires on HUP/INT/TERM so an interrupted run rolls back instead of
# leaving a half-activated state.
rolled_back=0
rollback() {
[ "$rolled_back" -eq 1 ] && return 0
rolled_back=1
echo "rsmon-worker activation failed; restoring the prior install" >&2
stop_svc || true
rm -f "$binary.new" "$env_file.new" "$unit_file.new" "$pid_file" || true
if [ "$had_binary" -eq 1 ]; then
cp -p "$backup_dir/binary" "$binary" || true
elif [ -e "$binary" ]; then
rm -f "$binary" || true
fi
if [ "$had_env" -eq 1 ]; then
cp -p "$backup_dir/worker.env" "$env_file" || true
chmod 0600 "$env_file" || true
elif [ -e "$env_file" ]; then
rm -f "$env_file" || true
fi
if [ -n "$unit_file" ]; then
if [ "$had_unit" -eq 1 ]; then
cp -p "$backup_dir/unit" "$unit_file" || true
chmod "$unit_mode" "$unit_file" || true
set_unit_enabled "$unit_enabled" || true
else
set_unit_enabled 0 || true
rm -f "$unit_file" || true
fi
fi
rm -f "$env_tmp" "$unit_tmp" || true
rm -rf "$backup_dir" || true
if [ "$had_binary" -eq 1 ] && [ "$had_env" -eq 1 ] && [ "$no_start" -ne 1 ]; then
restart_svc || true
fi
release_lock
exit 1
}
trap rollback EXIT HUP INT TERM
# --- atomic binary install ------------------------------------------------
mkdir -p "$(dirname "$binary")"
install -m 0755 "$stage" "$binary.new"
mv -f "$binary.new" "$binary"
# --- atomic env install (secrets, mode 0600) ------------------------------
mkdir -p "$config_dir"
chmod 0750 "$config_dir"
install -m 0600 "$env_tmp" "$env_file.new"
mv -f "$env_file.new" "$env_file"
rm -f "$env_tmp"
# --- data directory -------------------------------------------------------
mkdir -p "$data_dir" "$data_dir/webapp"
chmod 0755 "$data_dir" "$data_dir/webapp"
# --- service definition ---------------------------------------------------
if [ -n "$unit_file" ] && [ -f "$unit_tmp" ]; then
mkdir -p "$(dirname "$unit_file")"
install -m "$unit_mode" "$unit_tmp" "$unit_file.new"
mv -f "$unit_file.new" "$unit_file"
rm -f "$unit_tmp"
set_unit_enabled 1
fi
# --- start and verify -----------------------------------------------------
# A failed start or a worker that does not answer /healthz triggers the
# rollback trap.
if [ "$no_start" -ne 1 ]; then
if ! restart_svc; then
echo "rsmon-worker activation failed: process or /healthz verification failed" >&2
exit 1
fi
else
stop_svc || true
fi
# --- success --------------------------------------------------------------
rm -rf "$backup_dir"
rm -f "$env_tmp" "$unit_tmp"
release_lock
trap - EXIT HUP INT TERM
if [ "$no_start" -eq 1 ]; then
echo "rsmon-worker activated: binary=$binary supervisor=$supervisor unit=${unit_file:-none} started=no"
else
echo "rsmon-worker activated: binary=$binary supervisor=$supervisor unit=${unit_file:-none} started=yes"
fi
exit 0
`
// activateParams are the resolved, controller-validated values fed into
// the remote activation script.
type activateParams struct {
Stage string
Binary string
ConfigDir string
EnvFile string
EnvTmp string
DataDir string
UnitTmp string
UnitFile string
UnitMode string
UnitName string // systemd unit name (rsmon-worker.service)
RCName string // rc-service name (rsmon-worker)
Supervisor string // systemd | openrc | none
NoStart string // 0 or 1
}
// activateWorker performs the work-package-4 activation for an already
// staged source build over the live SSH executor. It uses the env bytes
// rendered once during option normalization, uploads the env and service
// definition into a server-created 0700 temp dir (no /tmp symlink/TOCTOU
// attack surface), runs the atomic activation script, and records where
// and how the worker was activated. Every remote temp file it creates is
// removed on success and failure.
func (e *sourceExecutor) activateWorker(opts SourceInstallOptions, res *SourceInstallResult) error {
a := opts.Activation
name := strings.TrimSpace(a.Name)
p := resolvePaths(name)
envData := opts.activationEnv
if len(envData) == 0 {
return errors.New("activation environment was not rendered (internal error)")
}
unitContent, unitFile, unitMode := unitContent(res.Detection.InitSystem, p)
svcName := strings.TrimSuffix(p.unitName, ".service")
supOut, err := e.runPlain("sh -c " + shellQuote(runningInitScript))
if err != nil {
return fmt.Errorf("detect running init system: %w", err)
}
supervisor := normalizeSupervisor(string(supOut))
// Create a server-side 0700 temp dir owned by the SSH user, then
// upload the env/unit into it. mktemp -d produces an unpredictable,
// private path, so a hostile local user cannot pre-create a symlink at
// a predictable /tmp name (the classic upload TOCTOU). The token
// never appears in argv or logs: it travels only as base64 over the
// session stdin and later only inside the mode-0600 env file.
out, err := e.runPlain("d=$(mktemp -d /tmp/rsmon-worker-act.XXXXXX) && chmod 0700 \"$d\" && echo \"$d\"")
if err != nil {
return fmt.Errorf("create secure upload directory: %w", err)
}
secureDir := strings.TrimSpace(string(out))
if secureDir == "" || !strings.HasPrefix(secureDir, "/tmp/") {
return errors.New("remote returned an invalid secure upload directory")
}
envTmp := secureDir + "/worker.env"
unitTmp := secureDir + "/unit"
if err := uploadBytes(e.client, envData, envTmp, 0o600); err != nil {
return fmt.Errorf("upload worker environment: %w", err)
}
if unitContent != "" {
if err := uploadBytes(e.client, []byte(unitContent), unitTmp, 0o644); err != nil {
return fmt.Errorf("upload service definition: %w", err)
}
}
defer func() {
_ = runRemote(e.client, "rm -rf -- "+shellQuote(secureDir), nil)
}()
noStart := "0"
if a.NoStart {
noStart = "1"
}
script := activateScript(activateParams{
Stage: res.StageBinary,
Binary: p.binary,
ConfigDir: p.configDir,
EnvFile: p.envFile,
EnvTmp: envTmp,
DataDir: p.dataDir,
UnitTmp: unitTmp,
UnitFile: unitFile,
UnitMode: unitMode,
UnitName: p.unitName,
RCName: svcName,
Supervisor: supervisor,
NoStart: noStart,
})
if _, err := e.runPrivileged("sh -c " + shellQuote(script)); err != nil {
return fmt.Errorf("activate worker service: %w", err)
}
res.Activation = &ActivationResult{
Binary: p.binary,
ConfigDir: p.configDir,
EnvFile: p.envFile,
DataDir: p.dataDir,
UnitFile: unitFile,
UnitName: p.unitName,
Supervisor: supervisor,
Started: !a.NoStart,
}
return nil
}
// validateRenderedEnv runs the strict environment-file parser over the
// rendered env bytes in memory so the exact bytes written remotely pass
// the same gate as a user-supplied file, without re-reading a file.
func validateRenderedEnv(data []byte) error {
_, err := parseEnvironmentContent(data)
return err
}
// openrcInitFor renders the native OpenRC init script for an instance. It
// mirrors the hardening of the systemd unit (data dir, env file, webapp
// data dir) using OpenRC conventions; it is written for real hosts where
// openrc is the running init, while containers that cannot run openrc
// fall back to the embedded supervisor.
func openrcInitFor(p paths) string {
description := "RSMon distributed monitoring worker"
svcName := strings.TrimSuffix(p.unitName, ".service")
if p.name != "" {
description += " (" + p.name + ")"
}
return fmt.Sprintf(`#!/sbin/openrc-run
# Managed by the rsmon-worker source installer; do not edit by hand.
name=%s
description=%s
command=%s
command_background=true
pidfile=%s/worker.pid
output_log=%s/worker.log
error_log=%s/worker.log
depend() {
need net
}
start_pre() {
checkpath --directory --mode 0755 --owner root:root %s %s/webapp
if [ -f %s ]; then
# OpenRC runs the command with the init script's environment, so
# the env-file variables must be exported (set -a) or the worker
# would start without them.
set -a
. %s
set +a
fi
export HOME=%s
export RSMON_WEBAPP_DATA_DIR=%s/webapp
}
`, svcName, description, p.binary, p.dataDir, p.dataDir, p.dataDir,
p.dataDir, p.dataDir, p.envFile, p.envFile, p.dataDir, p.dataDir)
}

Просмотреть файл

@@ -0,0 +1,438 @@
package installer
import (
"os"
"os/exec"
"path/filepath"
"strings"
"testing"
)
// This file exercises the rendered activation script against stub
// systemd/OpenRC implementations ("targeted shell fixture tests"). Real
// systemd and OpenRC cannot run inside the Docker/OpenSSH harness (the
// container PID 1 is sshd), so these tests execute the actual remote
// script on the host with fake `systemctl`, `rc-service`, and `rc-update`
// binaries that simulate an init-managed host. This covers the unit
// install/enable, the supervisor-driven restart, the `is-active`-based
// health loop, the enable-state rollback, the activation lock, and the
// interrupted-run recovery paths that the E2E fixtures cannot reach.
//
// Residual limitation: the stub init tools verify command flow and state
// transitions, not the real systemd/OpenRC unit semantics; a real
// init-system smoke test remains out of scope for the container harness.
// systemctlStub records every invocation and simulates unit state in a
// fake state directory: `is-active`/`is-enabled` consult marker files,
// `restart` marks the unit active (or fails when the fail-restart marker
// is present), and `enable`/`disable` toggle the enabled marker.
const systemctlStub = `#!/bin/sh
echo "$*" >> "$SYSCTL_LOG"
action="$1"
shift
unit=""
for a in "$@"; do
case "$a" in --*) continue ;; esac
unit="$a"
break
done
case "$action" in
is-active) [ -f "$FAKE_SYSTEMD_DIR/$unit.active" ] || exit 3 ;;
is-enabled) [ -f "$FAKE_SYSTEMD_DIR/$unit.enabled" ] || exit 1 ;;
enable) touch "$FAKE_SYSTEMD_DIR/$unit.enabled"; exit 0 ;;
disable) rm -f "$FAKE_SYSTEMD_DIR/$unit.enabled"; exit 0 ;;
daemon-reload) exit 0 ;;
restart) [ -f "$FAKE_SYSTEMD_DIR/fail-restart" ] && exit 1
touch "$FAKE_SYSTEMD_DIR/$unit.active"; exit 0 ;;
stop) rm -f "$FAKE_SYSTEMD_DIR/$unit.active"; exit 0 ;;
*) exit 0 ;;
esac
`
// rcServiceStub simulates OpenRC service state; rcUpdateStub simulates
// runlevel enablement.
const rcServiceStub = `#!/bin/sh
echo "$*" >> "$RC_LOG"
svc="$1"
action="$2"
case "$action" in
restart) [ -f "$FAKE_RC_DIR/fail-restart" ] && exit 1
touch "$FAKE_RC_DIR/$svc.active"; exit 0 ;;
status) [ -f "$FAKE_RC_DIR/$svc.active" ] || exit 1 ;;
stop) rm -f "$FAKE_RC_DIR/$svc.active"; exit 0 ;;
*) exit 0 ;;
esac
`
const rcUpdateStub = `#!/bin/sh
echo "$*" >> "$RCU_LOG"
case "$1" in
add) touch "$FAKE_RC_DIR/$2.enabled"; exit 0 ;;
del) rm -f "$FAKE_RC_DIR/$2.enabled"; exit 0 ;;
*) exit 0 ;;
esac
`
// workerStub stands in for the built worker binary: it satisfies the
// `--version` validation and the `/healthz` liveness probe.
const workerStub = `#!/bin/sh
if [ "$1" = "--version" ]; then
echo "rsmon-worker version=dev commit=deadbeefdead buildDate=2026-01-01T00:00:00Z"
exit 0
fi
if [ "$1" = "liveness" ]; then
exit 0
fi
exit 1
`
// initShellFixture builds a temp "host" with stub init tools and a stub
// worker, plus the uploaded env/unit the activation script consumes.
type initShellFixture struct {
root string
binDir string
sysdDir string
rcDir string
sysctlLog string
rcLog string
rcuLog string
params activateParams
}
func writeExec(t *testing.T, path, content string) {
t.Helper()
if err := os.WriteFile(path, []byte(content), 0o700); err != nil {
t.Fatal(err)
}
}
func newInitShellFixture(t *testing.T, supervisor string) *initShellFixture {
t.Helper()
root := t.TempDir()
fx := &initShellFixture{
root: root,
binDir: filepath.Join(root, "bin"),
sysdDir: filepath.Join(root, "sysd"),
rcDir: filepath.Join(root, "rc"),
sysctlLog: filepath.Join(root, "sysctl.log"),
rcLog: filepath.Join(root, "rc.log"),
rcuLog: filepath.Join(root, "rcu.log"),
}
for _, d := range []string{
fx.binDir, fx.sysdDir, fx.rcDir, filepath.Join(root, "upload"),
filepath.Join(root, "usr/local/bin"), filepath.Join(root, "etc/rsmon-worker"),
} {
if err := os.MkdirAll(d, 0o755); err != nil {
t.Fatal(err)
}
}
writeExec(t, filepath.Join(fx.binDir, "systemctl"), systemctlStub)
writeExec(t, filepath.Join(fx.binDir, "rc-service"), rcServiceStub)
writeExec(t, filepath.Join(fx.binDir, "rc-update"), rcUpdateStub)
stage := filepath.Join(root, "usr/local/bin/rsmon-worker.stage")
writeExec(t, stage, workerStub)
envTmp := filepath.Join(root, "upload", "worker.env")
unitTmp := filepath.Join(root, "upload", "unit")
if err := os.WriteFile(envTmp, []byte(
"RSMON_URL=https://rsmon.ru\nRSMON_TOKEN=shell-test-token\nWORKER_HOST=127.0.0.1\nWORKER_PORT=27401\n",
), 0o600); err != nil {
t.Fatal(err)
}
unitFile := filepath.Join(root, "etc/systemd/system/rsmon-worker.service")
unitMode := "0644"
if supervisor == "openrc" {
unitFile = filepath.Join(root, "etc/init.d/rsmon-worker")
unitMode = "0755"
if err := os.MkdirAll(filepath.Dir(unitFile), 0o755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(unitTmp, []byte("# fake openrc unit\n"), 0o755); err != nil {
t.Fatal(err)
}
} else {
if err := os.MkdirAll(filepath.Dir(unitFile), 0o755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(unitTmp, []byte("[Unit]\n# fake systemd unit\n"), 0o644); err != nil {
t.Fatal(err)
}
}
dataDir := filepath.Join(root, "var/lib/rsmon-worker")
fx.params = activateParams{
Stage: stage,
Binary: filepath.Join(root, "usr/local/bin/rsmon-worker"),
ConfigDir: filepath.Join(root, "etc/rsmon-worker"),
EnvFile: filepath.Join(root, "etc/rsmon-worker/worker.env"),
EnvTmp: envTmp,
DataDir: dataDir,
UnitTmp: unitTmp,
UnitFile: unitFile,
UnitMode: unitMode,
UnitName: "rsmon-worker.service",
RCName: "rsmon-worker",
Supervisor: supervisor,
NoStart: "0",
}
return fx
}
// run executes the rendered activation script with the stub init tools
// first in PATH.
func (fx *initShellFixture) run(t *testing.T) (string, error) {
t.Helper()
script := activateScript(fx.params)
cmd := exec.Command("sh", "-c", script)
cmd.Env = append(
os.Environ(),
"PATH="+fx.binDir+":"+os.Getenv("PATH"),
"FAKE_SYSTEMD_DIR="+fx.sysdDir,
"FAKE_RC_DIR="+fx.rcDir,
"SYSCTL_LOG="+fx.sysctlLog,
"RC_LOG="+fx.rcLog,
"RCU_LOG="+fx.rcuLog,
)
out, err := cmd.CombinedOutput()
return string(out), err
}
func (fx *initShellFixture) log(t *testing.T, path string) string {
t.Helper()
b, err := os.ReadFile(path)
if err != nil {
return ""
}
return string(b)
}
func (fx *initShellFixture) assertNoLeftovers(t *testing.T) {
t.Helper()
for _, name := range []string{".rsmon-backup", ".rsmon-activate.lock"} {
if _, err := os.Stat(filepath.Join(fx.params.DataDir, name)); !os.IsNotExist(err) {
t.Fatalf("%s left behind after activation", name)
}
}
}
// TestActivateShellSystemd runs the real activation script against a stub
// systemd host and verifies the init-managed flow: the unit is installed
// and enabled, restart goes through systemctl, and the health loop uses
// `systemctl is-active` (no pid file is ever created).
func TestActivateShellSystemd(t *testing.T) {
fx := newInitShellFixture(t, "systemd")
out, err := fx.run(t)
if err != nil {
t.Fatalf("systemd activation failed: %v\n%s", err, out)
}
if !strings.Contains(out, "rsmon-worker activated") {
t.Fatalf("no activation success line:\n%s", out)
}
unit, err := os.ReadFile(fx.params.UnitFile)
if err != nil {
t.Fatalf("unit not installed: %v", err)
}
if !strings.Contains(string(unit), "[Unit]") {
t.Fatalf("installed unit has wrong content: %q", unit)
}
st, err := os.Stat(fx.params.EnvFile)
if err != nil {
t.Fatalf("env not installed: %v", err)
}
if st.Mode().Perm() != 0o600 {
t.Fatalf("env mode = %v, want 0600", st.Mode().Perm())
}
log := fx.log(t, fx.sysctlLog)
for _, want := range []string{
"enable rsmon-worker.service",
"restart rsmon-worker.service",
"is-active --quiet rsmon-worker.service",
} {
if !strings.Contains(log, want) {
t.Fatalf("systemctl log missing %q:\n%s", want, log)
}
}
// The health loop used systemctl is-active, never the embedded
// supervisor's pid file.
if _, err := os.Stat(filepath.Join(fx.params.DataDir, "worker.pid")); !os.IsNotExist(err) {
t.Fatalf("systemd activation created a pid file; the health loop must use systemctl is-active")
}
if _, err := os.Stat(filepath.Join(fx.sysdDir, "rsmon-worker.service.enabled")); err != nil {
t.Fatalf("unit not enabled: %v", err)
}
fx.assertNoLeftovers(t)
}
// TestActivateShellOpenRC runs the real activation script against a stub
// OpenRC host: the init script is installed executable, the runlevel
// enablement and rc-service restart/status are used, and the health loop
// goes through rc-service (no pid file).
func TestActivateShellOpenRC(t *testing.T) {
fx := newInitShellFixture(t, "openrc")
out, err := fx.run(t)
if err != nil {
t.Fatalf("openrc activation failed: %v\n%s", err, out)
}
if !strings.Contains(out, "rsmon-worker activated") {
t.Fatalf("no activation success line:\n%s", out)
}
st, err := os.Stat(fx.params.UnitFile)
if err != nil {
t.Fatalf("openrc init not installed: %v", err)
}
if st.Mode().Perm() != 0o755 {
t.Fatalf("openrc init mode = %v, want 0755", st.Mode().Perm())
}
rcLog := fx.log(t, fx.rcLog)
for _, want := range []string{"rsmon-worker restart", "rsmon-worker status"} {
if !strings.Contains(rcLog, want) {
t.Fatalf("rc-service log missing %q:\n%s", want, rcLog)
}
}
rcuLog := fx.log(t, fx.rcuLog)
if !strings.Contains(rcuLog, "add rsmon-worker default") {
t.Fatalf("rc-update add not issued:\n%s", rcuLog)
}
if _, err := os.Stat(filepath.Join(fx.rcDir, "rsmon-worker.enabled")); err != nil {
t.Fatalf("service not enabled in the runlevel: %v", err)
}
if _, err := os.Stat(filepath.Join(fx.params.DataDir, "worker.pid")); !os.IsNotExist(err) {
t.Fatalf("openrc activation created a pid file; the health loop must use rc-service status")
}
fx.assertNoLeftovers(t)
}
// TestActivateShellSystemdRollbackFreshDisablesUnit proves a failed
// activation on a fresh host disables the newly-enabled unit and removes
// the unit file (no dangling enablement).
func TestActivateShellSystemdRollbackFreshDisablesUnit(t *testing.T) {
fx := newInitShellFixture(t, "systemd")
if err := os.WriteFile(filepath.Join(fx.sysdDir, "fail-restart"), []byte(""), 0o644); err != nil {
t.Fatal(err)
}
out, err := fx.run(t)
if err == nil {
t.Fatal("activation succeeded despite restart failure")
}
if !strings.Contains(out, "start failure") {
t.Fatalf("no start-failure classification:\n%s", out)
}
log := fx.log(t, fx.sysctlLog)
if !strings.Contains(log, "disable rsmon-worker.service") {
t.Fatalf("fresh rollback did not disable the unit:\n%s", log)
}
if _, err := os.Stat(fx.params.UnitFile); !os.IsNotExist(err) {
t.Fatal("unit file left behind after fresh rollback")
}
if _, err := os.Stat(filepath.Join(fx.sysdDir, "rsmon-worker.service.enabled")); !os.IsNotExist(err) {
t.Fatal("unit still enabled after fresh rollback")
}
fx.assertNoLeftovers(t)
}
// TestActivateShellSystemdRollbackRestoresEnableState proves a failed
// activation on a host with a prior enabled unit restores the unit
// byte-for-byte and re-applies the prior enabled state.
func TestActivateShellSystemdRollbackRestoresEnableState(t *testing.T) {
fx := newInitShellFixture(t, "systemd")
if err := os.WriteFile(fx.params.UnitFile, []byte("PRIOR-UNIT-CONTENT"), 0o644); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(fx.sysdDir, "rsmon-worker.service.enabled"), []byte(""), 0o644); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(fx.sysdDir, "fail-restart"), []byte(""), 0o644); err != nil {
t.Fatal(err)
}
out, err := fx.run(t)
if err == nil {
t.Fatal("activation succeeded despite restart failure")
}
if !strings.Contains(out, "start failure") {
t.Fatalf("no start-failure classification:\n%s", out)
}
unit, err := os.ReadFile(fx.params.UnitFile)
if err != nil {
t.Fatal(err)
}
if string(unit) != "PRIOR-UNIT-CONTENT" {
t.Fatalf("prior unit not restored: %q", unit)
}
log := fx.log(t, fx.sysctlLog)
if !strings.Contains(log, "enable rsmon-worker.service") {
t.Fatalf("prior enabled state not re-applied:\n%s", log)
}
if _, err := os.Stat(filepath.Join(fx.sysdDir, "rsmon-worker.service.enabled")); err != nil {
t.Fatalf("unit not re-enabled after rollback: %v", err)
}
fx.assertNoLeftovers(t)
}
// TestActivateShellStaleLockBroken proves a leftover lock from a killed
// run (dead pid) is broken automatically and the activation proceeds.
func TestActivateShellStaleLockBroken(t *testing.T) {
fx := newInitShellFixture(t, "systemd")
lockDir := filepath.Join(fx.params.DataDir, ".rsmon-activate.lock")
if err := os.MkdirAll(lockDir, 0o700); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(lockDir, "pid"), []byte("999999\n"), 0o644); err != nil {
t.Fatal(err)
}
out, err := fx.run(t)
if err != nil {
t.Fatalf("activation failed with a stale lock: %v\n%s", err, out)
}
if !strings.Contains(out, "rsmon-worker activated") {
t.Fatalf("no activation success line:\n%s", out)
}
if _, err := os.Stat(lockDir); !os.IsNotExist(err) {
t.Fatal("lock not released after activation")
}
}
// TestActivateShellRecoveryFromInterruptedBackup proves the interrupted-run
// recovery: a leftover backup marker from a killed activation is restored
// before the fresh run, so a later validation failure still leaves the
// recovered prior install in place.
func TestActivateShellRecoveryFromInterruptedBackup(t *testing.T) {
fx := newInitShellFixture(t, "none")
backupDir := filepath.Join(fx.params.DataDir, ".rsmon-backup")
if err := os.MkdirAll(backupDir, 0o700); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(backupDir, "binary"), []byte("RECOVERED-BINARY"), 0o755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(backupDir, "worker.env"), []byte("RSMON_URL=https://rsmon.ru\nRSMON_TOKEN=test\n"), 0o600); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(backupDir, ".marker"), []byte("0"), 0o600); err != nil {
t.Fatal(err)
}
// Corrupt the stage so the fresh activation fails at validation (before
// the fresh snapshot), proving the recovered files were already in place.
if err := os.WriteFile(fx.params.Stage, []byte("not a script"), 0o644); err != nil {
t.Fatal(err)
}
out, err := fx.run(t)
if err == nil {
t.Fatal("activation succeeded with a corrupt stage")
}
if !strings.Contains(out, "recovering interrupted activation") {
t.Fatalf("recovery did not run:\n%s", out)
}
b, err := os.ReadFile(fx.params.Binary)
if err != nil {
t.Fatalf("recovered binary not restored: %v", err)
}
if string(b) != "RECOVERED-BINARY" {
t.Fatalf("recovered binary content = %q", b)
}
if _, err := os.Stat(backupDir); !os.IsNotExist(err) {
t.Fatal("backup not consumed by recovery")
}
}

409
internal/installer/sourceactivate_test.go Обычный файл
Просмотреть файл

@@ -0,0 +1,409 @@
package installer
import (
"os"
"path/filepath"
"strings"
"testing"
"rocketgit.ru/rsmon/worker/internal/sshinstall"
)
func TestActivateScriptMarkers(t *testing.T) {
script := activateScript(activateParams{
Stage: "/opt/rsmon-worker-src/rsmon-worker",
Binary: "/usr/local/bin/rsmon-worker",
ConfigDir: "/etc/rsmon-worker",
EnvFile: "/etc/rsmon-worker/worker.env",
EnvTmp: "/tmp/rsmon-worker-abc.env",
DataDir: "/var/lib/rsmon-worker",
UnitTmp: "/tmp/rsmon-worker-abc.service",
UnitFile: "/etc/systemd/system/rsmon-worker.service",
UnitMode: "0644",
UnitName: "rsmon-worker.service",
RCName: "rsmon-worker",
Supervisor: "none",
NoStart: "0",
})
for _, want := range []string{
"set -eu",
// staged binary validated before any state is touched
"\"$stage\" --version >/dev/null",
// snapshot + rollback
"backup_dir=\"$data_dir/.rsmon-backup\"",
"rm -rf \"$backup_dir\"",
"cp -p \"$binary\" \"$backup_dir/binary\"",
"cp -p \"$env_file\" \"$backup_dir/worker.env\"",
"printf '%s\\n' \"$unit_enabled\" > \"$backup_dir/.marker\"",
"trap rollback EXIT HUP INT TERM",
"restoring the prior install",
// atomic installs with the right perms
"install -m 0755 \"$stage\" \"$binary.new\"",
"mv -f \"$binary.new\" \"$binary\"",
"install -m 0600 \"$env_tmp\" \"$env_file.new\"",
"mv -f \"$env_file.new\" \"$env_file\"",
"chmod 0750 \"$config_dir\"",
"mkdir -p \"$data_dir\" \"$data_dir/webapp\"",
"install -m \"$unit_mode\" \"$unit_tmp\" \"$unit_file.new\"",
"mv -f \"$unit_file.new\" \"$unit_file\"",
// supervisor + pid + health verification
"pid_file=\"$data_dir/worker.pid\"",
"env -i PATH=\"/usr/bin:/bin:/sbin:/usr/sbin\"",
"RSMON_WORKER_ENV_FILE=\"$env_file\"",
". \"$RSMON_WORKER_ENV_FILE\"",
"exec \"$@\"",
"nohup \"$1\" >>\"$2\" 2>&1 & echo $! > \"$3\"",
"run_env \"$binary\" liveness >/dev/null 2>&1",
"while [ \"$i\" -lt 30 ]; do",
"rsmon-worker activated:",
// activation lock + interrupted-run recovery
"lock_dir=\"$data_dir/.rsmon-activate.lock\"",
"acquire_lock",
"another rsmon-worker activation is in progress",
"recovering interrupted activation from $backup_dir",
".marker",
// pid belongs to the expected binary before kill
"readlink \"/proc/$1/exe\"",
"\"$binary (deleted)\"",
"process_is_worker \"$pid\"",
// rollback preserves metadata and enable state
"cp -p \"$backup_dir/binary\" \"$binary\"",
"cp -p \"$backup_dir/worker.env\" \"$env_file\"",
"set_unit_enabled \"$unit_enabled\"",
"systemctl is-enabled",
// temp/backup cleanup on success
"rm -rf \"$backup_dir\"",
"release_lock",
"trap - EXIT HUP INT TERM",
} {
if !strings.Contains(script, want) {
t.Fatalf("activate script missing %q:\n%s", want, script)
}
}
// The rollback must be armed only after the snapshot so a corrupt
// staging binary (validated first) never triggers a destructive
// rollback of the prior install.
if !strings.Contains(script, "\"$stage\" --version") || !strings.Contains(script, "trap rollback EXIT HUP INT TERM") {
t.Fatalf("activate script must validate the staged binary before arming rollback:\n%s", script)
}
stageCheck := strings.Index(script, "\"$stage\" --version")
trapIdx := strings.Index(script, "trap rollback EXIT HUP INT TERM")
if stageCheck < 0 || trapIdx < stageCheck {
t.Fatalf("staged-binary validation must precede the rollback trap:\n%s", script)
}
}
func TestActivateScriptSystemdSupervisor(t *testing.T) {
script := activateScript(activateParams{
Stage: "/opt/rsmon-worker-src/rsmon-worker",
Binary: "/usr/local/bin/rsmon-worker",
ConfigDir: "/etc/rsmon-worker",
EnvFile: "/etc/rsmon-worker/worker.env",
EnvTmp: "/tmp/e.env",
DataDir: "/var/lib/rsmon-worker",
UnitTmp: "/tmp/u.service",
UnitFile: "/etc/systemd/system/rsmon-worker.service",
UnitMode: "0644",
UnitName: "rsmon-worker.service",
RCName: "rsmon-worker",
Supervisor: "systemd",
NoStart: "0",
})
for _, want := range []string{
"supervisor=systemd",
"systemctl daemon-reload",
"systemctl restart \"$unit_name\"",
"systemctl is-active --quiet \"$unit_name\"",
"systemctl enable \"$unit_name\"",
"svc_active()",
"start failure: systemctl restart $unit_name failed",
"start failure: worker service is not active",
} {
if !strings.Contains(script, want) {
t.Fatalf("systemd supervisor missing %q:\n%s", want, script)
}
}
// The systemd health loop must use `systemctl is-active`, never the
// embedded supervisor's pid file.
if !strings.Contains(script, "systemd) systemctl is-active --quiet \"$unit_name\"") {
t.Fatalf("systemd health loop must use systemctl is-active:\n%s", script)
}
}
func TestActivateScriptOpenRCSupervisor(t *testing.T) {
script := activateScript(activateParams{
Stage: "/opt/rsmon-worker-src/rsmon-worker",
Binary: "/usr/local/bin/rsmon-worker",
ConfigDir: "/etc/rsmon-worker",
EnvFile: "/etc/rsmon-worker/worker.env",
EnvTmp: "/tmp/e.env",
DataDir: "/var/lib/rsmon-worker",
UnitTmp: "/tmp/u",
UnitFile: "/etc/init.d/rsmon-worker",
UnitMode: "0755",
UnitName: "rsmon-worker.service",
RCName: "rsmon-worker",
Supervisor: "openrc",
NoStart: "0",
})
for _, want := range []string{
"rc-service \"$rc_name\" restart",
"rc-service \"$rc_name\" status",
"rc-update add \"$rc_name\" default",
"start failure: rc-service restart $rc_name failed",
"rc-update del \"$rc_name\" default",
} {
if !strings.Contains(script, want) {
t.Fatalf("openrc supervisor missing %q:\n%s", want, script)
}
}
}
func TestActivateScriptNoStartSkipsRestart(t *testing.T) {
script := activateScript(activateParams{
Stage: "/opt/rsmon-worker-src/rsmon-worker", Binary: "/usr/local/bin/rsmon-worker",
ConfigDir: "/etc/rsmon-worker", EnvFile: "/etc/rsmon-worker/worker.env", EnvTmp: "/tmp/e.env",
DataDir: "/var/lib/rsmon-worker", UnitTmp: "", UnitFile: "", UnitMode: "",
UnitName: "rsmon-worker.service", RCName: "rsmon-worker", Supervisor: "none", NoStart: "1",
})
if !strings.Contains(script, "[ \"$no_start\" -ne 1 ]") {
t.Fatalf("no-start gate missing:\n%s", script)
}
if !strings.Contains(script, "started=no") {
t.Fatalf("no-start summary missing:\n%s", script)
}
}
func TestActivateScriptNeverContainsSecrets(t *testing.T) {
const secret = "super-secret-token-value"
params := activateParams{
Stage: "/opt/rsmon-worker-src/rsmon-worker", Binary: "/usr/local/bin/rsmon-worker",
ConfigDir: "/etc/rsmon-worker", EnvFile: "/etc/rsmon-worker/worker.env", EnvTmp: "/tmp/e.env",
DataDir: "/var/lib/rsmon-worker", UnitTmp: "/tmp/u", UnitFile: "/etc/systemd/system/rsmon-worker.service",
UnitMode: "0644", UnitName: "rsmon-worker.service", RCName: "rsmon-worker",
Supervisor: "systemd", NoStart: "0",
}
script := activateScript(params)
if strings.Contains(script, secret) {
t.Fatalf("activate script contains a secret:\n%s", script)
}
// The env file is referenced by path; its contents (the secrets) are
// only sourced at runtime and never echoed.
for _, want := range []string{"RSMON_TOKEN=", secret} {
if strings.Contains(script, want) {
t.Fatalf("activate script must not embed env contents (%q):\n%s", want, script)
}
}
}
func TestOpenRCInit(t *testing.T) {
unit := openrcInitFor(resolvePaths(""))
for _, want := range []string{
"#!/sbin/openrc-run",
"name=rsmon-worker",
"description=RSMon distributed monitoring worker",
"command=/usr/local/bin/rsmon-worker",
"command_background=true",
"pidfile=/var/lib/rsmon-worker/worker.pid",
"output_log=/var/lib/rsmon-worker/worker.log",
"need net",
". /etc/rsmon-worker/worker.env",
"set -a",
"set +a",
"RSMON_WEBAPP_DATA_DIR=/var/lib/rsmon-worker/webapp",
} {
if !strings.Contains(unit, want) {
t.Fatalf("openrc init missing %q:\n%s", want, unit)
}
}
named := openrcInitFor(resolvePaths("edge"))
for _, want := range []string{
"name=rsmon-worker-edge", "command=/usr/local/bin/rsmon-worker-edge",
"pidfile=/var/lib/rsmon-worker-edge/worker.pid", "/etc/rsmon-worker-edge/worker.env", "(edge)",
} {
if !strings.Contains(named, want) {
t.Fatalf("named openrc init missing %q:\n%s", want, named)
}
}
}
func TestRunningInitScript(t *testing.T) {
for _, want := range []string{"set -eu", "/run/systemd/system", "systemctl", "/run/openrc/softlevel", "rc-service", "echo none"} {
if !strings.Contains(runningInitScript, want) {
t.Fatalf("running-init script missing %q:\n%s", want, runningInitScript)
}
}
}
func TestNormalizeSupervisor(t *testing.T) {
for _, in := range []string{"systemd", "openrc", "systemd\n", " openrc "} {
want := strings.TrimSpace(in)
if got := normalizeSupervisor(in); got != want {
t.Fatalf("normalizeSupervisor(%q) = %q, want %q", in, got, want)
}
}
for _, in := range []string{"", "none", "sysvinit", " "} {
if got := normalizeSupervisor(in); got != "none" {
t.Fatalf("normalizeSupervisor(%q) = %q, want none", in, got)
}
}
}
func TestUnitContent(t *testing.T) {
p := resolvePaths("")
systemd, file, mode := unitContent(sshinstall.InitSystemd, p)
if systemd == "" || file != "/etc/systemd/system/rsmon-worker.service" || mode != "0644" {
t.Fatalf("systemd unit content = %q, %q, %q", systemd, file, mode)
}
if !strings.Contains(systemd, "ExecStart=/usr/local/bin/rsmon-worker\n") {
t.Fatalf("systemd unit not the classic hardened unit:\n%s", systemd)
}
openrc, file, mode := unitContent(sshinstall.InitOpenRC, p)
if openrc == "" || file != "/etc/init.d/rsmon-worker" || mode != "0755" {
t.Fatalf("openrc unit content = %q, %q, %q", openrc, file, mode)
}
if content, file, mode := unitContent(sshinstall.InitUnknown, p); content != "" || file != "" || mode != "" {
t.Fatalf("unknown-init unit content = %q, %q, %q, want empty (no-service gate)", content, file, mode)
}
}
func TestRenderEnvActivation(t *testing.T) {
t.Setenv("RSMON_URL", "")
t.Setenv("RSMON_TOKEN", "")
t.Setenv("WORKER_HOST", "")
t.Setenv("WORKER_PORT", "")
t.Setenv("WORKER_LOGIN", "")
t.Setenv("WORKER_PASSWORD", "")
data, err := (ActivationOptions{Activate: true, URL: "https://rsmon.ru", Token: "secret"}).renderEnv()
if err != nil {
t.Fatal(err)
}
for _, want := range []string{"RSMON_URL=https://rsmon.ru\n", "RSMON_TOKEN=secret\n", "WORKER_HOST=127.0.0.1\n", "WORKER_PORT=27401\n"} {
if !strings.Contains(string(data), want) {
t.Fatalf("rendered env missing %q:\n%s", want, data)
}
}
// PUBLIC_URL canonicalization is reused from the classic installer.
data, err = (ActivationOptions{
Activate: true, URL: "https://rsmon.ru", Token: "secret",
PublicURL: "https://worker.example.com",
}).renderEnv()
if err != nil {
t.Fatal(err)
}
if !strings.Contains(string(data), "PUBLIC_URL=https://worker.example.com\n") {
t.Fatalf("rendered env missing PUBLIC_URL:\n%s", data)
}
}
func TestRenderEnvActivationRejectsMixedBasicAuth(t *testing.T) {
t.Setenv("RSMON_TOKEN", "")
if _, err := (ActivationOptions{
Activate: true, URL: "https://rsmon.ru", Token: "secret",
Login: "admin",
}).renderEnv(); err == nil {
t.Fatal("login-only basic auth accepted")
}
}
func TestRenderEnvActivationRequiresToken(t *testing.T) {
if _, err := (ActivationOptions{Activate: true, URL: "https://rsmon.ru"}).renderEnv(); err == nil {
t.Fatal("activation without a token accepted")
}
if _, err := (ActivationOptions{Activate: true, Token: "x", URL: "not-a-url"}).renderEnv(); err == nil {
t.Fatal("activation with a malformed URL accepted")
}
if _, err := (ActivationOptions{Activate: true, Token: "x", URL: "https://rsmon.ru", Name: "Bad_Name"}).renderEnv(); err == nil {
t.Fatal("activation with an invalid instance name accepted")
}
if _, err := (ActivationOptions{Activate: true, Token: "x", URL: "https://rsmon.ru", Name: "edge"}).renderEnv(); err == nil {
t.Fatal("named activation without WORKER_PORT accepted")
}
}
// TestRenderEnvActivationReadsEnvFileOnce proves the env file is read a
// single time and validated from the in-memory bytes: swapping the file
// after the read cannot smuggle a different value into the render, and a
// malformed file fails on the read bytes.
func TestRenderEnvActivationReadsEnvFileOnce(t *testing.T) {
t.Setenv("RSMON_TOKEN", "")
path := filepath.Join(t.TempDir(), "worker.env")
if err := os.WriteFile(path, []byte("RSMON_URL=https://rsmon.ru\nRSMON_TOKEN=file-token\nWORKER_PORT=28888\n"), 0o600); err != nil {
t.Fatal(err)
}
data, err := (ActivationOptions{Activate: true, EnvFile: path}).renderEnv()
if err != nil {
t.Fatal(err)
}
if !strings.Contains(string(data), "RSMON_TOKEN=file-token\n") || !strings.Contains(string(data), "WORKER_PORT=28888\n") {
t.Fatalf("env file values not rendered:\n%s", data)
}
// A malformed file must be rejected from the same read.
if err := os.WriteFile(path, []byte("RSMON_URL=https://rsmon.ru\nRSMON_TOKEN=has space\n"), 0o600); err != nil {
t.Fatal(err)
}
if _, err := (ActivationOptions{Activate: true, EnvFile: path}).renderEnv(); err == nil {
t.Fatal("malformed env file accepted")
}
}
// TestNormalizeSourceOptionsActivationRendersOnce verifies the rendered
// env is computed during normalization and reused, so a later call cannot
// re-read a changed env file.
func TestNormalizeSourceOptionsActivationRendersOnce(t *testing.T) {
t.Setenv("RSMON_TOKEN", "")
path := filepath.Join(t.TempDir(), "worker.env")
if err := os.WriteFile(path, []byte("RSMON_URL=https://rsmon.ru\nRSMON_TOKEN=once-token\n"), 0o600); err != nil {
t.Fatal(err)
}
base := SourceInstallOptions{
SSHOptions: SSHOptions{Host: "h", User: "u"},
Activation: ActivationOptions{Activate: true, EnvFile: path},
}
norm, err := normalizeSourceOptions(base)
if err != nil {
t.Fatal(err)
}
if !strings.Contains(string(norm.activationEnv), "RSMON_TOKEN=once-token\n") {
t.Fatalf("activation env not cached during normalization: %q", norm.activationEnv)
}
// Even after the file changes, the cached render is authoritative.
if err := os.WriteFile(path, []byte("RSMON_URL=https://evil.test\nRSMON_TOKEN=evil\n"), 0o600); err != nil {
t.Fatal(err)
}
if strings.Contains(string(norm.activationEnv), "evil") {
t.Fatalf("changed env file leaked into the cached render: %q", norm.activationEnv)
}
}
func TestNormalizeSourceOptionsActivation(t *testing.T) {
t.Setenv("RSMON_TOKEN", "")
base := SourceInstallOptions{SSHOptions: SSHOptions{Host: "h", User: "u"}}
if _, err := normalizeSourceOptions(base); err != nil {
t.Fatalf("staging-only options must stay valid: %v", err)
}
act := base
act.Activation = ActivationOptions{Activate: true, URL: "https://rsmon.ru"}
if _, err := normalizeSourceOptions(act); err == nil || !strings.Contains(err.Error(), "RSMON_TOKEN") {
t.Fatalf("activation without a token error = %v", err)
}
ok := base
ok.Activation = ActivationOptions{Activate: true, URL: "https://rsmon.ru", Token: "secret"}
if _, err := normalizeSourceOptions(ok); err != nil {
t.Fatalf("valid activation rejected: %v", err)
}
}
func TestValidateRenderedEnv(t *testing.T) {
if err := validateRenderedEnv([]byte("RSMON_URL=https://rsmon.ru\nRSMON_TOKEN=secret\n")); err != nil {
t.Fatal(err)
}
if err := validateRenderedEnv([]byte("RSMON_TOKEN=secret\n")); err == nil {
t.Fatal("env without RSMON_URL accepted")
}
if err := validateRenderedEnv([]byte("RSMON_URL=https://rsmon.ru\nRSMON_TOKEN=secret value\n")); err == nil {
t.Fatal("env with whitespace accepted")
}
}

Просмотреть файл

@@ -50,11 +50,23 @@ type SourceInstallOptions struct {
ToolchainDir string
// StageBinary is where the built worker binary is written. It must
// be absolute and defaults to <BuildDir>/rsmon-worker. The running
// service and its config are NOT touched by this work package.
// service and its config are NOT touched until Activation runs.
StageBinary string
// SessionTimeout bounds each remote command. 0 uses the default
// (30 minutes); the build step can legitimately run for minutes.
SessionTimeout time.Duration
// Activation drives work package 4: after the staging build, the
// staged binary, validated environment, data dir, and the detected
// init's service definition are installed atomically and the worker
// is started and verified (process + /healthz). Any failure rolls
// back to the prior working install. Empty keeps SourceInstall at the
// staging boundary and touches no service configuration.
Activation ActivationOptions
// activationEnv is the once-rendered activation environment, computed
// during option normalization so the env file is read and rendered
// exactly once per run (no TOCTOU between preflight and activation).
activationEnv []byte
}
// SourceInstallResult is what a source installation resolved to. The
@@ -70,6 +82,10 @@ type SourceInstallResult struct {
ResolvedCommit string
RecordFile string
StageBinary string
// Activation is set when the staged binary was atomically installed
// and the worker started/verified. It records the installed layout
// and the supervisor used. Nil when Activation was not requested.
Activation *ActivationResult
}
// defaultToolchainDir is the standard Go installation prefix.
@@ -142,14 +158,20 @@ func (e *sourceExecutor) fileProber() sshinstall.FileProber {
// branch, builds the worker to a staging path, and only then records the
// resolved branch and commit.
//
// The running service, its configuration, and its data directory are
// deliberately untouched: atomic activation and rollback are the next
// work package. Every remote step runs with the same privilege path as
// `deploy` (root, passwordless sudo, or sudo -S), every interpolated
// value is single-quoted, every step script fails closed (`set -eu` or
// explicit `&&`/retry), and every failure returns a bounded, actionable
// error. Each remote command is capped by SessionTimeout and its stdout
// is size-bounded.
// When Activation.Activate is set, the staged binary is then atomically
// installed together with the validated environment, data directory, and
// the detected init's service definition, and the worker is started and
// verified (process + /healthz); any activation/start/health failure
// rolls back to the prior working install. Without activation the
// running service, its configuration, and its data directory are
// deliberately untouched (the staging boundary of work package 3).
//
// Every remote step runs with the same privilege path as `deploy` (root,
// passwordless sudo, or sudo -S), every interpolated value is
// single-quoted, every step script fails closed (`set -eu` or explicit
// `&&`/retry), and every failure returns a bounded, actionable error.
// Each remote command is capped by SessionTimeout and its stdout is
// size-bounded.
func SourceInstall(opts SourceInstallOptions) (*SourceInstallResult, error) {
opts, err := normalizeSourceOptions(opts)
if err != nil {
@@ -276,6 +298,15 @@ func SourceInstall(opts SourceInstallOptions) (*SourceInstallResult, error) {
if _, err := executor.runPrivileged("sh -c " + shellQuote(commitRecordScript(plan.BuildDir, branch, commit))); err != nil {
return nil, fmt.Errorf("record resolved commit: %w", err)
}
// Work package 4: atomically install the staged build and activate
// the worker. On any failure the remote script restores the previous
// working install and this step returns a bounded error.
if opts.Activation.Activate {
if err := executor.activateWorker(opts, result); err != nil {
return nil, err
}
}
return result, nil
}
@@ -316,6 +347,20 @@ func normalizeSourceOptions(o SourceInstallOptions) (SourceInstallOptions, error
if o.SessionTimeout <= 0 {
o.SessionTimeout = defaultSessionTimeout
}
if o.Activation.Activate {
// Render (and validate) the activation environment exactly once
// here. The rendered bytes are reused at activation time, so the
// env file is read a single time and cannot change between the
// preflight and the remote install (env-file TOCTOU).
env, err := o.Activation.renderEnv()
if err != nil {
return o, fmt.Errorf("activation: %w", err)
}
if err := validateRenderedEnv(env); err != nil {
return o, fmt.Errorf("activation: %w", err)
}
o.activationEnv = env
}
return o, nil
}

Просмотреть файл

@@ -4,6 +4,7 @@ import (
"crypto/ed25519"
"crypto/rand"
"fmt"
"io"
"net"
"strings"
"sync"
@@ -106,7 +107,18 @@ func (s *fakeSSHServer) handleConn(conn net.Conn, config *ssh.ServerConfig) {
}
go func() {
defer channel.Close()
// Drain the client's stdin so uploads (which stream base64
// over the session stdin) never block the channel window.
// The channel is not closed until the client's write side is
// exhausted, mirroring the real server behavior.
var drained sync.WaitGroup
drained.Add(1)
go func() {
defer drained.Done()
_, _ = io.Copy(io.Discard, channel)
}()
s.handleSession(channel, requests)
drained.Wait()
}()
}
}
@@ -151,9 +163,10 @@ func (s *fakeSSHServer) execCommand(channel ssh.Channel, command string) {
// defaultFakeExec simulates a minimal Linux host: it answers os-release,
// uname, the init-marker file probe, origin URL, branch resolution, and
// rev-parse, and accepts every install step. A pinned-branch existence
// check (show-ref) fails by default so the missing-branch path is
// exercised without extra setup.
// rev-parse, accepts every install step, and simulates the work-package-4
// activation (running-init probe, env/unit uploads, and the activate
// script succeeding). A pinned-branch existence check (show-ref) fails by
// default so the missing-branch path is exercised without extra setup.
func defaultFakeExec(command string) (string, string, int) {
switch {
case strings.Contains(command, "cat /etc/os-release"):
@@ -170,6 +183,18 @@ func defaultFakeExec(command string) (string, string, int) {
return fakeCommitHex + "\n", "", 0
case strings.Contains(command, "show-ref"):
return "", "branch not found", 1
case strings.Contains(command, "softlevel"):
// running-init probe: no systemd/openrc is actually booted.
return "none\n", "", 0
case strings.Contains(command, "mktemp -d /tmp/rsmon-worker-act"):
// server-side secure 0700 upload dir.
return "/tmp/rsmon-worker-act-fake\n", "", 0
case strings.Contains(command, "base64 -d"):
// env/unit uploads succeed.
return "", "", 0
case strings.Contains(command, "rsmon-worker activated"):
// the activate script completed successfully.
return "", "", 0
default:
return "", "", 0
}
@@ -184,9 +209,25 @@ func testSSHOptions(port int) SourceInstallOptions {
Password: fakeSSHPassword,
InsecureHostKey: true,
},
// Staging-only by default so the work-package-3 orchestration
// tests keep their historical shape; activation tests opt in.
Activation: ActivationOptions{Activate: false},
}
}
// testActivationOptions wraps testSSHOptions with the credentials and
// activation flag needed to run the work-package-4 flow against the fake
// server.
func testActivationOptions(port int) SourceInstallOptions {
opts := testSSHOptions(port)
opts.Activation = ActivationOptions{
Activate: true,
URL: "https://rsmon.ru",
Token: fakeSSHPassword + "-token",
}
return opts
}
func TestSourceInstallSSHFlow(t *testing.T) {
srv := startFakeSSHServer(t, nil)
res, err := SourceInstall(testSSHOptions(srv.Port()))
@@ -349,3 +390,147 @@ func TestSourceInstallSSHCommandTimeout(t *testing.T) {
t.Fatalf("err = %v, want command timeout", err)
}
}
// TestSourceInstallSSHActivationFlow verifies the work-package-4 flow
// over a real SSH session: after the staging build and commit record, the
// installer probes the running init, uploads the env and service
// definition, runs the atomic activation script, and reports the installed
// layout. The secrets-absent contract holds: the worker token never
// reaches a remote command.
func TestSourceInstallSSHActivationFlow(t *testing.T) {
srv := startFakeSSHServer(t, nil)
res, err := SourceInstall(testActivationOptions(srv.Port()))
if err != nil {
t.Fatal(err)
}
if res.Activation == nil {
t.Fatal("activation result missing")
}
if res.Activation.Binary != "/usr/local/bin/rsmon-worker" ||
res.Activation.EnvFile != "/etc/rsmon-worker/worker.env" ||
res.Activation.DataDir != "/var/lib/rsmon-worker" ||
res.Activation.UnitFile != "/etc/systemd/system/rsmon-worker.service" ||
res.Activation.Supervisor != "none" ||
!res.Activation.Started {
t.Fatalf("activation result = %+v", res.Activation)
}
commands := srv.Commands()
var joined strings.Builder
for _, c := range commands {
joined.WriteString(c)
joined.WriteString("\n")
}
for _, want := range []string{
"/run/systemd/system",
"softlevel",
"mktemp -d /tmp/rsmon-worker-act",
"umask 077; base64 -d >",
"rsmon-worker activated",
"install -m 0755",
"install -m 0600",
"chmod 0750",
"worker.pid",
"liveness",
"rm -rf --",
} {
if !strings.Contains(joined.String(), want) {
t.Fatalf("activation commands missing %q:\n%s", want, joined.String())
}
}
// Ordering: build and commit record, then a server-side secure upload
// dir is created, the env/unit are uploaded into it, and the activation
// script runs last.
idx := func(sub string) int {
for i, c := range commands {
if strings.Contains(c, sub) {
return i
}
}
t.Fatalf("command %q not found in %v", sub, commands)
return -1
}
if idx("branch=%s") >= idx("mktemp -d /tmp/rsmon-worker-act") ||
idx("mktemp -d /tmp/rsmon-worker-act") >= idx("base64 -d >") ||
idx("base64 -d >") >= idx("rsmon-worker activated") {
t.Fatalf("activation order wrong: %v", commands)
}
if strings.Contains(joined.String(), fakeSSHPassword+"-token") {
t.Fatal("worker token leaked into a remote command")
}
}
// TestSourceInstallSSHNoActivationSkipsService proves the staging
// boundary: without Activation the installer never probes the init system,
// uploads an env/unit, or runs the activation script.
func TestSourceInstallSSHNoActivationSkipsService(t *testing.T) {
srv := startFakeSSHServer(t, nil)
res, err := SourceInstall(testSSHOptions(srv.Port()))
if err != nil {
t.Fatal(err)
}
if res.Activation != nil {
t.Fatalf("activation must not run without the flag: %+v", res.Activation)
}
joined := strings.Join(srv.Commands(), "\n")
for _, forbidden := range []string{"softlevel", "base64 -d >", "rsmon-worker activated", "worker.pid"} {
if strings.Contains(joined, forbidden) {
t.Fatalf("staging-only flow ran activation step %q:\n%s", forbidden, joined)
}
}
}
// TestSourceInstallSSHActivationRequiresToken verifies the credentials
// gate: activation without a token fails before any remote connection.
func TestSourceInstallSSHActivationRequiresToken(t *testing.T) {
srv := startFakeSSHServer(t, nil)
opts := testSSHOptions(srv.Port())
opts.Activation = ActivationOptions{Activate: true, URL: "https://rsmon.ru"}
_, err := SourceInstall(opts)
if err == nil || !strings.Contains(err.Error(), "RSMON_TOKEN") {
t.Fatalf("err = %v, want token requirement", err)
}
if got := len(srv.Commands()); got != 0 {
t.Fatalf("commands ran before validation failed: %d", got)
}
}
// TestSourceInstallSSHActivationFailure verifies the bounded failure
// path: a failing activate script surfaces as a labelled error with the
// remote stderr, and the uploaded temp env/unit are cleaned up by the
// controller defer regardless of the outcome.
func TestSourceInstallSSHActivationFailure(t *testing.T) {
srv := startFakeSSHServer(t, func(command string) (string, string, int) {
if strings.Contains(command, "rsmon-worker activated") {
return "", "activation exploded: disk full", 1
}
return defaultFakeExec(command)
})
_, err := SourceInstall(testActivationOptions(srv.Port()))
if err == nil || !strings.Contains(err.Error(), "activate worker service") || !strings.Contains(err.Error(), "disk full") {
t.Fatalf("err = %v, want bounded activation failure", err)
}
joined := strings.Join(srv.Commands(), "\n")
if !strings.Contains(joined, "rm -rf --") {
t.Fatalf("secure upload dir cleanup not issued after activation failure:\n%s", joined)
}
}
// TestSourceInstallSSHActivationHealthFailure verifies the health gate
// surfaces as an activation failure and the temp files are still cleaned.
// The fake cannot run the shell script, so the failure is injected at the
// activate-command level; the real rollback semantics are covered by the
// Docker/OpenSSH E2E tests.
func TestSourceInstallSSHActivationHealthFailure(t *testing.T) {
srv := startFakeSSHServer(t, func(command string) (string, string, int) {
if strings.Contains(command, "rsmon-worker activated") {
return "", "health failure: worker did not answer /healthz", 1
}
return defaultFakeExec(command)
})
_, err := SourceInstall(testActivationOptions(srv.Port()))
if err == nil || !strings.Contains(err.Error(), "did not answer /healthz") {
t.Fatalf("err = %v, want /healthz verification failure", err)
}
}