feat(installer): activate source builds atomically
Все проверки выполнены успешно
CI / test (push) Successful in 4m30s
Docker / Build and publish worker image (push) Successful in 17m26s
Все проверки выполнены успешно
CI / test (push) Successful in 4m30s
Docker / Build and publish worker image (push) Successful in 17m26s
Этот коммит содержится в:
36
README.md
36
README.md
@@ -183,11 +183,18 @@ A Go SSH source installer is implemented as the `source-install` subcommand: it
|
|||||||
detects the remote distro and architecture over the existing SSH transport,
|
detects the remote distro and architecture over the existing SSH transport,
|
||||||
installs the minimal build prerequisites, downloads and SHA-256-verifies the
|
installs the minimal build prerequisites, downloads and SHA-256-verifies the
|
||||||
pinned Go 1.26 toolchain, clones/updates the public repository, records the
|
pinned Go 1.26 toolchain, clones/updates the public repository, records the
|
||||||
resolved branch and commit, and builds the worker to a staging path. It
|
resolved branch and commit, builds the worker to a staging path, and then
|
||||||
deliberately stops before touching the running service or its configuration
|
atomically activates the staged binary together with the validated environment,
|
||||||
(atomic activation and rollback are the next milestone). The pure
|
data directory, and the detected init's service definition. It starts the
|
||||||
detection/planning layer and the Docker/OpenSSH test harness that accepts it
|
worker and verifies the process and `/healthz`; any activation, start, or
|
||||||
are implemented; see
|
health failure rolls back to the previous working install (preserving the
|
||||||
|
prior binary/env/unit and its enable state), concurrent activations are
|
||||||
|
serialized by a lock, a run killed mid-flight is recovered from its
|
||||||
|
leftover backup marker on the next activation, and reruns are idempotent
|
||||||
|
(exactly one worker process, no leaked temp files). Pass `--no-activate`
|
||||||
|
to stop at the staging build, and `--no-start` to install the layout
|
||||||
|
without starting the worker. The pure detection/planning layer and the
|
||||||
|
Docker/OpenSSH test harness that accepts it are implemented; see
|
||||||
[`docs/source-installation.md`](docs/source-installation.md); run the live
|
[`docs/source-installation.md`](docs/source-installation.md); run the live
|
||||||
fixture matrix with `make test-ssh`.
|
fixture matrix with `make test-ssh`.
|
||||||
|
|
||||||
@@ -195,7 +202,8 @@ fixture matrix with `make test-ssh`.
|
|||||||
./bin/rsmon-worker source-install \
|
./bin/rsmon-worker source-install \
|
||||||
--host worker.example.com \
|
--host worker.example.com \
|
||||||
--user deploy \
|
--user deploy \
|
||||||
--identity-file ~/.ssh/id_ed25519
|
--identity-file ~/.ssh/id_ed25519 \
|
||||||
|
--token-file ./worker.token
|
||||||
```
|
```
|
||||||
|
|
||||||
By default the installer builds the remote's default branch and records what it
|
By default the installer builds the remote's default branch and records what it
|
||||||
@@ -204,13 +212,21 @@ resolves to (the public repository currently publishes `master`). Pass
|
|||||||
install fails before building. The repository must be an `https://` URL without
|
install fails before building. The repository must be an `https://` URL without
|
||||||
userinfo.
|
userinfo.
|
||||||
|
|
||||||
The built binary is left at `/opt/rsmon-worker-src/rsmon-worker` (override with
|
The built binary is staged at `/opt/rsmon-worker-src/rsmon-worker` (override with
|
||||||
`--build-dir` / `--stage-binary`), the toolchain at `/usr/local/go` (replaced
|
`--build-dir` / `--stage-binary`), the toolchain at `/usr/local/go` (replaced
|
||||||
atomically: download, verify, stage, swap with rollback), and the resolved
|
atomically: download, verify, stage, swap with rollback), and the resolved
|
||||||
branch and commit in `/opt/rsmon-worker-src/rsmon-worker.commit` (written only
|
branch and commit in `/opt/rsmon-worker-src/rsmon-worker.commit` (written only
|
||||||
after a successful build). The same SSH auth, secret-file, and host-key options
|
after a successful build). Activation installs the binary to
|
||||||
as `deploy` apply; no worker token is sent because the source install does not
|
`/usr/local/bin/rsmon-worker`, the environment to
|
||||||
configure a service. Prefer `--key-passphrase-file`, `--password-file`, and
|
`/etc/rsmon-worker/worker.env` (mode 0600), the data directory to
|
||||||
|
`/var/lib/rsmon-worker`, and a systemd unit or OpenRC init script for the
|
||||||
|
detected init (an embedded supervisor manages the process when no init is
|
||||||
|
running, as in containers). The env and unit are uploaded into a server-side
|
||||||
|
0700 `mktemp -d` directory so no local user can race a predictable `/tmp` path,
|
||||||
|
and the env file is read/rendered exactly once. The same SSH auth,
|
||||||
|
secret-file, and host-key options as `deploy` apply; the worker token is
|
||||||
|
supplied with `--token`/`--token-file` and written only to the mode-0600 env
|
||||||
|
file. Prefer `--key-passphrase-file`, `--password-file`, and
|
||||||
`--sudo-password-file` over their direct-flag equivalents: file options keep
|
`--sudo-password-file` over their direct-flag equivalents: file options keep
|
||||||
secrets out of the process list and shell history.
|
secrets out of the process list and shell history.
|
||||||
|
|
||||||
|
|||||||
@@ -156,16 +156,21 @@ func secretValue(direct, path string) (string, error) {
|
|||||||
return strings.TrimRight(string(b), "\r\n"), nil
|
return strings.TrimRight(string(b), "\r\n"), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// sourceInstallCommand drives the remote source installer (work package
|
// sourceInstallCommand drives the remote source installer
|
||||||
// 3 of docs/source-installation.md): prerequisites, verified Go
|
// (docs/source-installation.md): prerequisites, verified Go toolchain,
|
||||||
// toolchain, clone/update, resolved branch/commit record, and a staging
|
// clone/update, resolved branch/commit record, a staging build, and (work
|
||||||
// build. It reuses the deploy SSH options; it never touches the running
|
// package 4) the atomic activation of the staged binary, validated
|
||||||
// service, configuration, or data directory on the remote host.
|
// environment, data dir, and the detected init's service definition,
|
||||||
|
// followed by a start and a process + /healthz verification. It reuses
|
||||||
|
// the deploy SSH options. Any activation/start/health failure rolls back
|
||||||
|
// to the prior working install; --no-activate keeps the staging-only
|
||||||
|
// behavior.
|
||||||
func sourceInstallCommand(args []string) int {
|
func sourceInstallCommand(args []string) int {
|
||||||
fs := flag.NewFlagSet("source-install", flag.ContinueOnError)
|
fs := flag.NewFlagSet("source-install", flag.ContinueOnError)
|
||||||
fs.SetOutput(os.Stderr)
|
fs.SetOutput(os.Stderr)
|
||||||
var opts installer.SourceInstallOptions
|
var opts installer.SourceInstallOptions
|
||||||
var passphraseFile, passwordFile, sudoPasswordFile string
|
var passphraseFile, passwordFile, sudoPasswordFile, tokenFile, workerPasswordFile string
|
||||||
|
var noActivate bool
|
||||||
fs.StringVar(&opts.Host, "host", "", "SSH server hostname or address")
|
fs.StringVar(&opts.Host, "host", "", "SSH server hostname or address")
|
||||||
fs.IntVar(&opts.Port, "port", 22, "SSH server port")
|
fs.IntVar(&opts.Port, "port", 22, "SSH server port")
|
||||||
fs.StringVar(&opts.User, "user", "", "SSH username")
|
fs.StringVar(&opts.User, "user", "", "SSH username")
|
||||||
@@ -188,8 +193,22 @@ func sourceInstallCommand(args []string) int {
|
|||||||
fs.StringVar(&opts.ToolchainDir, "toolchain-dir", "", "remote Go install path ending in /go (default /usr/local/go)")
|
fs.StringVar(&opts.ToolchainDir, "toolchain-dir", "", "remote Go install path ending in /go (default /usr/local/go)")
|
||||||
fs.StringVar(&opts.StageBinary, "stage-binary", "", "staging binary path (default <build-dir>/rsmon-worker)")
|
fs.StringVar(&opts.StageBinary, "stage-binary", "", "staging binary path (default <build-dir>/rsmon-worker)")
|
||||||
fs.DurationVar(&opts.SessionTimeout, "session-timeout", 0, "per-remote-command timeout (default 30m; 0 uses the default)")
|
fs.DurationVar(&opts.SessionTimeout, "session-timeout", 0, "per-remote-command timeout (default 30m; 0 uses the default)")
|
||||||
|
fs.BoolVar(&noActivate, "no-activate", false, "stop after the staging build and do not install/start a service (activation is the default)")
|
||||||
|
fs.StringVar(&opts.Activation.Name, "name", "", "instance name: activates rsmon-worker-<name> with its own config/data/unit/port")
|
||||||
|
fs.StringVar(&opts.Activation.URL, "url", installer.DefaultURL, "RSMon server URL (RSMON_URL)")
|
||||||
|
fs.StringVar(&opts.Activation.Token, "token", "", "worker API token (RSMON_TOKEN; required for activation)")
|
||||||
|
fs.StringVar(&opts.Activation.Token, "api-key", "", "worker API token (alias for --token)")
|
||||||
|
fs.StringVar(&tokenFile, "token-file", "", "file containing the worker API token")
|
||||||
|
fs.StringVar(&opts.Activation.PublicURL, "public-url", "", "advertised public origin (PUBLIC_URL; scheme + host, no path)")
|
||||||
|
fs.StringVar(&opts.Activation.Host, "worker-host", "", "worker webapp bind address (WORKER_HOST; default 127.0.0.1)")
|
||||||
|
fs.StringVar(&opts.Activation.Port, "worker-port", "", "worker webapp bind port (WORKER_PORT; default 27401, required with --name)")
|
||||||
|
fs.StringVar(&opts.Activation.Login, "worker-login", "", "operator console login (WORKER_LOGIN; default admin with a generated password)")
|
||||||
|
fs.StringVar(&opts.Activation.Password, "worker-password", "", "operator console password (WORKER_PASSWORD)")
|
||||||
|
fs.StringVar(&workerPasswordFile, "worker-password-file", "", "file containing the operator console password")
|
||||||
|
fs.StringVar(&opts.Activation.EnvFile, "env-file", "", "local systemd-safe env file uploaded for activation (overrides the individual knobs)")
|
||||||
|
fs.BoolVar(&opts.Activation.NoStart, "no-start", false, "install the binary, env, data dir, and service definition without starting the worker")
|
||||||
fs.Usage = func() {
|
fs.Usage = func() {
|
||||||
fmt.Fprintln(fs.Output(), "Usage: rsmon-worker source-install --host HOST --user USER [SSH options] [source options]")
|
fmt.Fprintln(fs.Output(), "Usage: rsmon-worker source-install --host HOST --user USER [--token TOKEN|--token-file FILE] [SSH options] [source options]")
|
||||||
fs.PrintDefaults()
|
fs.PrintDefaults()
|
||||||
}
|
}
|
||||||
if err := fs.Parse(args); err != nil {
|
if err := fs.Parse(args); err != nil {
|
||||||
@@ -215,6 +234,15 @@ func sourceInstallCommand(args []string) int {
|
|||||||
fmt.Fprintf(os.Stderr, "source-install failed: %v\n", err)
|
fmt.Fprintf(os.Stderr, "source-install failed: %v\n", err)
|
||||||
return 1
|
return 1
|
||||||
}
|
}
|
||||||
|
if opts.Activation.Token, err = secretValue(opts.Activation.Token, tokenFile); err != nil {
|
||||||
|
fmt.Fprintf(os.Stderr, "source-install failed: %v\n", err)
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
if opts.Activation.Password, err = secretValue(opts.Activation.Password, workerPasswordFile); err != nil {
|
||||||
|
fmt.Fprintf(os.Stderr, "source-install failed: %v\n", err)
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
opts.Activation.Activate = !noActivate
|
||||||
res, err := installer.SourceInstall(opts)
|
res, err := installer.SourceInstall(opts)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
fmt.Fprintf(os.Stderr, "source-install failed: %v\n", err)
|
fmt.Fprintf(os.Stderr, "source-install failed: %v\n", err)
|
||||||
@@ -227,6 +255,22 @@ func sourceInstallCommand(args []string) int {
|
|||||||
fmt.Printf(" commit: %s\n", res.ResolvedCommit)
|
fmt.Printf(" commit: %s\n", res.ResolvedCommit)
|
||||||
fmt.Printf(" record file: %s\n", res.RecordFile)
|
fmt.Printf(" record file: %s\n", res.RecordFile)
|
||||||
fmt.Printf(" staged build: %s\n", res.StageBinary)
|
fmt.Printf(" staged build: %s\n", res.StageBinary)
|
||||||
fmt.Println(" status: not installed as a service (activation is the next milestone)")
|
if res.Activation != nil {
|
||||||
|
fmt.Printf(" installed to: %s\n", res.Activation.Binary)
|
||||||
|
fmt.Printf(" env file: %s (mode 0600)\n", res.Activation.EnvFile)
|
||||||
|
fmt.Printf(" data dir: %s\n", res.Activation.DataDir)
|
||||||
|
if res.Activation.UnitFile != "" {
|
||||||
|
fmt.Printf(" service: %s (%s)\n", res.Activation.UnitFile, res.Activation.UnitName)
|
||||||
|
} else {
|
||||||
|
fmt.Println(" service: none (no supported init system detected; supervisor-managed)")
|
||||||
|
}
|
||||||
|
if res.Activation.Started {
|
||||||
|
fmt.Printf(" status: running (supervisor=%s, /healthz verified)\n", res.Activation.Supervisor)
|
||||||
|
} else {
|
||||||
|
fmt.Printf(" status: installed (not started, --no-start)\n")
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
fmt.Println(" status: not installed as a service (--no-activate)")
|
||||||
|
}
|
||||||
return 0
|
return 0
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -2,10 +2,7 @@
|
|||||||
|
|
||||||
## Status
|
## Status
|
||||||
|
|
||||||
In progress. Work package 1 (reusable Docker/OpenSSH harness and distro
|
In progress. Work packages 1-4 are implemented:
|
||||||
fixtures), the pure detection/planning foundation (work package 2 core),
|
|
||||||
and work package 3 (remote execution through the existing SSH transport)
|
|
||||||
are implemented:
|
|
||||||
|
|
||||||
- `internal/installer/harness` builds and runs real OpenSSH containers
|
- `internal/installer/harness` builds and runs real OpenSSH containers
|
||||||
for Alpine, Ubuntu, and Arch, waits for real network readiness, captures
|
for Alpine, Ubuntu, and Arch, waits for real network readiness, captures
|
||||||
@@ -24,17 +21,21 @@ are implemented:
|
|||||||
downloads and SHA-256-verifies the pinned Go toolchain before
|
downloads and SHA-256-verifies the pinned Go toolchain before
|
||||||
extraction, clones/updates the public repository, checks out the
|
extraction, clones/updates the public repository, checks out the
|
||||||
resolved branch, records the resolved branch and commit, and builds the
|
resolved branch, records the resolved branch and commit, and builds the
|
||||||
worker to a staging path. It deliberately does not install or replace
|
worker to a staging path.
|
||||||
the running service, configuration, or data directory: atomic
|
- Work package 4 (`installer.SourceInstall` with `Activation` enabled)
|
||||||
activation and rollback are the next work package.
|
atomically installs the staged binary, validated environment, data
|
||||||
|
directory, and the detected init's service definition; starts/restarts
|
||||||
|
the worker; verifies the process and `/healthz`; and rolls back to the
|
||||||
|
prior working install on any activation/start/health failure. Reruns
|
||||||
|
are idempotent with exactly one running worker and no leaked temp
|
||||||
|
files. The CLI runs the full flow by default and accepts
|
||||||
|
`--no-activate` (staging only) and `--no-start`.
|
||||||
|
|
||||||
The current Go installer can still only upload a binary or deploy an
|
The current Go installer can also upload a binary or deploy an immutable
|
||||||
immutable Docker image over SSH. Source installs now build remotely to a
|
Docker image over SSH (`deploy`). Source installs build remotely, then
|
||||||
staging path, but service activation over SSH (work package 4) is not
|
activate the staged build atomically with rollback. Existing tests are
|
||||||
implemented yet; the acceptance test stops after install-to-staging and
|
unit tests plus the harness tests that run against live OpenSSH
|
||||||
idempotent-rerun assertions. Existing tests are unit tests plus the
|
containers when explicitly enabled.
|
||||||
harness tests that run against live OpenSSH containers when explicitly
|
|
||||||
enabled.
|
|
||||||
|
|
||||||
## Initial Platform Scope
|
## Initial Platform Scope
|
||||||
|
|
||||||
@@ -66,7 +67,6 @@ It then:
|
|||||||
7. atomically installs the binary, validated environment, data directory, and
|
7. atomically installs the binary, validated environment, data directory, and
|
||||||
service definition;
|
service definition;
|
||||||
8. starts the service and verifies process status and `/healthz`.
|
8. starts the service and verifies process status and `/healthz`.
|
||||||
|
|
||||||
Repository, branch, Go version, checksum source, build directory, and Go module
|
Repository, branch, Go version, checksum source, build directory, and Go module
|
||||||
proxy may be configurable, but production output records their resolved values.
|
proxy may be configurable, but production output records their resolved values.
|
||||||
The default repository is publicly readable and requires no source credential.
|
The default repository is publicly readable and requires no source credential.
|
||||||
@@ -79,7 +79,7 @@ Work package 3 is `installer.SourceInstall` in
|
|||||||
verification) and runs every remote step with the same privilege path
|
verification) and runs every remote step with the same privilege path
|
||||||
(root, passwordless sudo, or `sudo -S -p ''` with the password delivered
|
(root, passwordless sudo, or `sudo -S -p ''` with the password delivered
|
||||||
only over stdin). Steps 1-6 of the flow above are implemented; step 7
|
only over stdin). Steps 1-6 of the flow above are implemented; step 7
|
||||||
(atomic install) is deliberately the next work package.
|
(atomic install) and step 8 (start + verify) are work package 4.
|
||||||
|
|
||||||
Per step:
|
Per step:
|
||||||
|
|
||||||
@@ -162,8 +162,89 @@ Security properties of work package 3:
|
|||||||
or `/usr/local/.go-staging-*`/`.go-backup` leaks after the rerun.
|
or `/usr/local/.go-staging-*`/`.go-backup` leaks after the rerun.
|
||||||
- Failed builds and failed checkouts leave the previous staging binary
|
- Failed builds and failed checkouts leave the previous staging binary
|
||||||
untouched (the binary is only overwritten by an atomically-swapped
|
untouched (the binary is only overwritten by an atomically-swapped
|
||||||
successful build, and a checkout failure aborts before the build);
|
successful build, and a checkout failure aborts before the build).
|
||||||
there is no running service or configuration to preserve yet.
|
|
||||||
|
## Work Package 4: Atomic Activation And Rollback
|
||||||
|
|
||||||
|
Work package 4 is the second half of `installer.SourceInstall`, driven by
|
||||||
|
`Activation` (`internal/installer/sourceactivate.go`). It runs over the
|
||||||
|
same SSH executor, privilege path, and bounded-error machinery as the
|
||||||
|
staging steps and installs the staged build atomically:
|
||||||
|
|
||||||
|
- **Layout.** The classic installer's on-disk layout is reused
|
||||||
|
(`resolvePaths`): binary `/usr/local/bin/rsmon-worker[-name]`, config
|
||||||
|
`/etc/rsmon-worker[-name]/worker.env` (mode 0600), data
|
||||||
|
`/var/lib/rsmon-worker[-name]`, and a service definition for the
|
||||||
|
detected init (systemd unit `/etc/systemd/system/rsmon-worker.service`
|
||||||
|
mode 0644, or an OpenRC script `/etc/init.d/rsmon-worker` mode 0755).
|
||||||
|
A named instance (`--name`) mirrors the classic multi-instance
|
||||||
|
convention with `-<name>` suffixes and its own unit and port.
|
||||||
|
- **Environment.** The env file is rendered by the classic installer's
|
||||||
|
`resolveInstallEnv` + `renderEnvFile` (so `PUBLIC_URL` canonicalization,
|
||||||
|
basic-auth XOR, required `RSMON_URL`/`RSMON_TOKEN`, and systemd-safe
|
||||||
|
value validation are identical to `install`). The env file is read
|
||||||
|
*exactly once* and validated from the in-memory bytes
|
||||||
|
(`parseEnvironmentContent`), and the rendered env is cached during
|
||||||
|
option normalization and reused at activation time, so a hostile local
|
||||||
|
writer cannot swap the file between the preflight and the remote
|
||||||
|
install (env-file TOCTOU). The rendered bytes are uploaded into a
|
||||||
|
server-created 0700 temp dir (`mktemp -d /tmp/rsmon-worker-act.XXXXXX`
|
||||||
|
under the SSH user) rather than a predictable `/tmp` path, eliminating
|
||||||
|
the symlink/TOCTOU attack surface on the upload while the token still
|
||||||
|
travels only as base64 over the session stdin and, later, only inside
|
||||||
|
the mode-0600 env file. The env is then installed with mode 0600 in the
|
||||||
|
config dir (mode 0750).
|
||||||
|
- **Atomic install.** The staged binary is validated (`<stage>
|
||||||
|
--version`) before any state is touched, then installed with a
|
||||||
|
same-directory temp file + rename. The env and unit files are installed
|
||||||
|
the same way. The data dir (and `webapp/` subdir) is created with mode
|
||||||
|
0755.
|
||||||
|
- **Supervisor.** `restart_svc` drives start/restart through whichever
|
||||||
|
init system is *actually running*: systemd (`systemctl`), OpenRC
|
||||||
|
(`rc-service`), or the embedded supervisor fallback when no init is
|
||||||
|
running (containers/chroots, the "no-service gate"). The health loop
|
||||||
|
verifies the service is active *through the supervisor* — `systemctl
|
||||||
|
is-active` for systemd, `rc-service status` for OpenRC, and a
|
||||||
|
zombie-aware pid check for the embedded supervisor — never through a
|
||||||
|
pid file on the init-managed paths. The embedded supervisor keeps the
|
||||||
|
worker as a single background process with a pid file under the data
|
||||||
|
dir, stops the previous instance before starting, refuses to kill a pid
|
||||||
|
whose executable is not the configured worker (`/proc/<pid>/exe`
|
||||||
|
checked before every `kill`), and answers `/healthz` via the worker's
|
||||||
|
own `liveness` subcommand with a clean `env -i` environment built from
|
||||||
|
the env file (so a rolled-back env can never leak a stale variable into
|
||||||
|
the restored process). The OpenRC init script exports the env-file
|
||||||
|
variables (`set -a` before sourcing) so the worker inherits them when
|
||||||
|
OpenRC starts it.
|
||||||
|
- **Rollback.** The prior binary, env, and unit (and the unit's enable
|
||||||
|
state) are snapshotted into the data dir before any mutation, and an
|
||||||
|
`EXIT`/`HUP`/`INT`/`TERM` trap restores them (preserving the prior
|
||||||
|
binary/env/unit metadata via `cp -p`) and restarts the prior service if
|
||||||
|
any later step fails or a signal interrupts the run. A corrupt staged
|
||||||
|
binary fails the pre-validation *before* the trap is armed, so the
|
||||||
|
prior install is never touched. A fresh-install failure disables the
|
||||||
|
newly-enabled unit (`systemctl disable` / `rc-update del`) and removes
|
||||||
|
it; a rerun failure restores the prior unit and re-applies its prior
|
||||||
|
enable state. Rollback also removes the backup dir, the `.new` temp
|
||||||
|
files, the uploaded env/unit temps, and the activation lock.
|
||||||
|
- **Lock, recovery, interruption.** A `mkdir`-based activation lock
|
||||||
|
(`.rsmon-activate.lock`, broken automatically when its recorded pid is
|
||||||
|
dead) prevents concurrent activations from racing the deployed-state
|
||||||
|
mutation. The snapshot is marked; a run killed mid-flight (SSH drop,
|
||||||
|
SIGKILL) leaves that marker, and the next activation restores the
|
||||||
|
leftover snapshot before proceeding, so the host never stays
|
||||||
|
half-activated.
|
||||||
|
- **Idempotency.** A rerun reuses the existing env (rewriting it
|
||||||
|
deterministically from the same knobs), swaps the binary atomically,
|
||||||
|
restarts exactly one worker, and leaves no `.rsmon-backup`, `.new`,
|
||||||
|
`/tmp/rsmon-worker-act.*`, or lock leftovers. `--no-start` installs the
|
||||||
|
full layout without starting anything and reruns stay idle.
|
||||||
|
|
||||||
|
The `source-install` CLI activates by default; pass `--no-activate` for
|
||||||
|
the staging-only behavior of work package 3. The worker token is required
|
||||||
|
for activation and is supplied with `--token`/`--token-file` or
|
||||||
|
`--env-file` (prefer the file options; direct flags expose the value
|
||||||
|
through the process list).
|
||||||
|
|
||||||
## Docker OpenSSH Test Harness
|
## Docker OpenSSH Test Harness
|
||||||
|
|
||||||
@@ -273,22 +354,37 @@ RSMON_TEST_DOCKER=1 go test -v -count=1 -timeout 30m ./internal/installer/harnes
|
|||||||
matches, the clone's origin is verified against the configured repository and
|
matches, the clone's origin is verified against the configured repository and
|
||||||
then fetched in place, the resolved branch/commit record is rewritten after
|
then fetched in place, the resolved branch/commit record is rewritten after
|
||||||
the new build succeeds, and the staging build atomically swaps over the
|
the new build succeeds, and the staging build atomically swaps over the
|
||||||
previous staging binary. The "one active service" guarantee is a work package
|
previous staging binary. Activation (work package 4) makes the rerun
|
||||||
4 property (activation); work package 3 leaves no running service to
|
idempotent at the service level too: exactly one worker process exists, the
|
||||||
duplicate and no leaked toolchain temp files.
|
env file is rewritten deterministically from the same knobs, and no
|
||||||
|
`.rsmon-backup`, `.new`, `/tmp/rsmon-worker-act.*`, or lock files leak.
|
||||||
- Wrong host fingerprints fail before remote mutation. The harness's fresh
|
- Wrong host fingerprints fail before remote mutation. The harness's fresh
|
||||||
known_hosts file is trust-on-first-use; its dedicated mismatch test dials
|
known_hosts file is trust-on-first-use; its dedicated mismatch test dials
|
||||||
against a known_hosts entry carrying a different server key and proves the
|
against a known_hosts entry carrying a different server key and proves the
|
||||||
dial fails before any command runs.
|
dial fails before any command runs.
|
||||||
- Tokens/passwords should come from files: `--key-passphrase-file`,
|
- Tokens/passwords should come from files: `--key-passphrase-file`,
|
||||||
`--password-file`, and `--sudo-password-file` keep secrets out of argv and
|
`--password-file`, `--sudo-password-file`, and the activation
|
||||||
shell history, while the equivalent direct flags expose them through the
|
`--token-file`/`--env-file` keep secrets out of argv and shell history,
|
||||||
process list. The source install itself sends no worker token or
|
while the equivalent direct flags expose them through the process list. The
|
||||||
control-plane credential at all, and sudo passwords travel only over the
|
worker token is written only to the mode-0600 env file, never echoed into a
|
||||||
session's stdin.
|
command or log, and sudo passwords travel only over the session's stdin. The
|
||||||
- Remote temporary files are removed on success and failure.
|
env and unit uploads land in a server-created 0700 `mktemp -d` directory (not
|
||||||
|
a predictable `/tmp` path), so no local user can plant a symlink at the
|
||||||
|
upload target (TOCTOU), and the env file itself is read/rendered exactly once
|
||||||
|
so a local writer cannot swap it between validation and upload.
|
||||||
|
- Remote temporary files are removed on success and failure, including the
|
||||||
|
activation backup dir, the uploaded env/unit temps, and the activation lock.
|
||||||
|
The snapshot is written under a marker so a run killed mid-flight is
|
||||||
|
recovered (restored) by the next activation instead of leaving the host
|
||||||
|
half-activated.
|
||||||
- Failed builds, failed checkouts, and failed toolchain swaps do not replace a
|
- Failed builds, failed checkouts, and failed toolchain swaps do not replace a
|
||||||
working binary or service definition.
|
working binary or service definition. Failed activations, failed starts, and
|
||||||
|
failed `/healthz` verifications restore the prior binary, env, and service
|
||||||
|
definition (preserving their metadata), re-apply the prior unit enable state
|
||||||
|
(or disable a freshly-enabled unit on a fresh failure), and bring the prior
|
||||||
|
worker back up (work package 4 rollback). The supervisor only ever kills a
|
||||||
|
process whose `/proc/<pid>/exe` matches the configured worker binary, so a
|
||||||
|
stale or recycled pid file can never kill an unrelated process.
|
||||||
- Package-manager and download failures return bounded actionable errors.
|
- Package-manager and download failures return bounded actionable errors.
|
||||||
- The installer verifies Go tarball checksum before extraction.
|
- The installer verifies Go tarball checksum before extraction.
|
||||||
|
|
||||||
@@ -300,24 +396,32 @@ RSMON_TEST_DOCKER=1 go test -v -count=1 -timeout 30m ./internal/installer/harnes
|
|||||||
- [x] 3. Execute source installation through the existing SSH transport
|
- [x] 3. Execute source installation through the existing SSH transport
|
||||||
(prerequisites, verified Go toolchain, clone/update, resolved branch and
|
(prerequisites, verified Go toolchain, clone/update, resolved branch and
|
||||||
commit record, and a build to a staging path; no service activation).
|
commit record, and a build to a staging path; no service activation).
|
||||||
- [ ] 4. Add atomic build/install, idempotency, and failure rollback.
|
- [x] 4. Add atomic build/install, idempotency, and failure rollback (binary,
|
||||||
|
env, data dir, and init service definition installed atomically; process
|
||||||
|
and `/healthz` verified; activation/start/health failures restore the
|
||||||
|
prior install; reruns keep exactly one service).
|
||||||
- [ ] 5. Add Alpine, Ubuntu, and Arch network E2E tests to CI.
|
- [ ] 5. Add Alpine, Ubuntu, and Arch network E2E tests to CI.
|
||||||
- [ ] 6. Add CentOS-family support.
|
- [ ] 6. Add CentOS-family support.
|
||||||
- [ ] 7. Plan native Windows service and macOS launchd installers separately.
|
- [ ] 7. Plan native Windows service and macOS launchd installers separately.
|
||||||
|
|
||||||
## Acceptance Gates
|
## Acceptance Gates
|
||||||
|
|
||||||
- [ ] All three initial Linux images install from a clean state through OpenSSH.
|
- [x] All three initial Linux images install from a clean state through OpenSSH:
|
||||||
Work package 3 covers the install-to-staging half (package install,
|
package install, verified Go 1.26, clone/update, resolved commit, staging
|
||||||
verified Go 1.26, clone/update, resolved commit, staging build); the
|
build, and (work package 4) atomic activation with a running verified
|
||||||
service-start half is the work package 4 gate.
|
worker.
|
||||||
- [ ] The built worker reports the expected version/commit and serves `/healthz`.
|
- [x] The built worker reports the expected version/commit and serves `/healthz`
|
||||||
- [ ] Re-running the installer succeeds without duplicate services or leaked files.
|
(verified by the E2E fixture tests and by the activation health gate).
|
||||||
- [ ] Host-key, checksum, clone, build, and service-start failure tests preserve the
|
- [x] Re-running the installer succeeds without duplicate services or leaked
|
||||||
previous installation.
|
files (exactly one worker process and no backup/temp leftovers asserted by
|
||||||
|
the rerun test on all three fixtures).
|
||||||
|
- [x] Host-key, checksum, clone, build, and service-start failure tests preserve
|
||||||
|
the previous installation (build/checkout, activation, start, and health
|
||||||
|
failure reruns on the Alpine fixture all restore the prior binary, env,
|
||||||
|
and running service).
|
||||||
- [ ] CI uses approved registry mirrors and cleans every test container/network.
|
- [ ] CI uses approved registry mirrors and cleans every test container/network.
|
||||||
|
|
||||||
## Verified Test Evidence (work packages 1-3)
|
## Verified Test Evidence (work packages 1-4)
|
||||||
|
|
||||||
Recorded 2026-08-12 from `make test-ssh` (Docker Engine 29.7.1):
|
Recorded 2026-08-12 from `make test-ssh` (Docker Engine 29.7.1):
|
||||||
|
|
||||||
@@ -334,14 +438,48 @@ Recorded 2026-08-12 from `make test-ssh` (Docker Engine 29.7.1):
|
|||||||
prerequisite install, verified Go 1.26 toolchain download/extraction, clone
|
prerequisite install, verified Go 1.26 toolchain download/extraction, clone
|
||||||
of the public repository, resolution of its default branch, resolved-commit
|
of the public repository, resolution of its default branch, resolved-commit
|
||||||
record file, and a staging build that reports the resolved commit via
|
record file, and a staging build that reports the resolved commit via
|
||||||
`--version`. The public repo's `HEAD` was `master` at `4651deb2...` during
|
`--version`. A rerun succeeds, keeps the same branch, reuses the toolchain,
|
||||||
the run, and all three fixtures resolved to that branch and commit (the
|
and leaves no `/tmp/rsmon-toolchain-*` temp dirs.
|
||||||
installer records whatever the remote publishes). A rerun succeeds, keeps
|
- `TestSourceInstallActivationFixtures` runs the full work-package-4 flow on
|
||||||
the same branch, reuses the toolchain, and leaves no
|
each fixture from a clean state: after the staging build the installer
|
||||||
`/tmp/rsmon-toolchain-*` temp dirs. The running service and its
|
atomically installs the binary (`/usr/local/bin/rsmon-worker`), env
|
||||||
configuration are not touched (work package 4 boundary).
|
(`/etc/rsmon-worker/worker.env`, mode 0600; config dir 0750), data dir
|
||||||
- The unit suite covers the per-step remote scripts, option validation,
|
(`/var/lib/rsmon-worker`), and the detected init's service definition
|
||||||
branch/commit parsing, sudo wrapping (password never in the command), the
|
(systemd unit on Ubuntu/Arch, OpenRC script on Alpine), then starts the
|
||||||
secrets-absent contract, and an in-process real-SSH orchestration flow with
|
worker via the embedded supervisor (no init runs inside the fixtures) and
|
||||||
failure paths for missing pinned branches, build failures, and detection
|
verifies exactly one worker process and a live `/healthz`. A rerun succeeds,
|
||||||
failures.
|
keeps a single process, rewrites the env deterministically, and leaves no
|
||||||
|
`.rsmon-backup`, `.new`, `/tmp/rsmon-worker-act.*`, or lock leftovers.
|
||||||
|
- `TestSourceInstallActivationFailureRollback` (Alpine) forces four rerun
|
||||||
|
failures and proves each restores the prior install byte-for-byte and
|
||||||
|
running: a dirty checkout (build failure), a sabotaged atomic swap
|
||||||
|
(activation failure), `WORKER_CLUSTER_ENABLED=true` without credentials (the
|
||||||
|
new process exits at boot; start failure), and `WORKER_HOST=255.255.255.255`
|
||||||
|
(the new process runs but `/healthz` is unreachable; health failure). In
|
||||||
|
every case the installed binary SHA-256, env, service definition, single
|
||||||
|
process, and liveness match the pre-failure state, and no backup, lock, or
|
||||||
|
temp files leak.
|
||||||
|
- `TestSourceInstallActivationNoStart` installs the full layout without
|
||||||
|
starting anything, and reruns stay idle.
|
||||||
|
- The targeted shell fixture tests (`TestActivateShell*`) execute the real
|
||||||
|
activation script against stub `systemctl`/`rc-service`/`rc-update`
|
||||||
|
implementations and cover the init-managed paths the Docker fixtures cannot
|
||||||
|
reach: systemd unit install/enable with a `systemctl is-active`-based health
|
||||||
|
loop (and no pid file), OpenRC install/enable with an `rc-service`-based
|
||||||
|
health loop, fresh-failure rollback disabling a newly-enabled unit, rerun
|
||||||
|
rollback restoring the prior unit and its enable state, stale-lock breaking,
|
||||||
|
and interrupted-run recovery from a leftover backup marker.
|
||||||
|
- The unit suite covers the per-step remote scripts, the activation script
|
||||||
|
(backup + recovery marker, activation lock, atomic install, rollback trap,
|
||||||
|
supervisor, pid/zombie handling, pid-belongs-to-binary checks, enable-state
|
||||||
|
restoration, secrets absent), OpenRC unit rendering (including env
|
||||||
|
export), option validation, branch/commit parsing, sudo wrapping (password
|
||||||
|
never in the command), and an in-process real-SSH orchestration flow with
|
||||||
|
failure paths for missing pinned branches, build failures, detection
|
||||||
|
failures, and activation/start/health failures.
|
||||||
|
- **Residual limitation (stated honestly):** live systemd and OpenRC cannot
|
||||||
|
run inside the Docker/OpenSSH harness (the container PID 1 is sshd), so the
|
||||||
|
init-managed supervisor paths are verified by the stub-based shell fixture
|
||||||
|
tests above rather than against a real init. The stub tools simulate unit
|
||||||
|
state and command flow, not real systemd/OpenRC unit semantics; a real
|
||||||
|
init-system smoke test on a booted host remains a follow-up.
|
||||||
|
|||||||
390
internal/installer/harness/integration_activation_test.go
Обычный файл
390
internal/installer/harness/integration_activation_test.go
Обычный файл
@@ -0,0 +1,390 @@
|
|||||||
|
package harness
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"golang.org/x/crypto/ssh"
|
||||||
|
|
||||||
|
"rocketgit.ru/rsmon/worker/internal/installer"
|
||||||
|
"rocketgit.ru/rsmon/worker/internal/sshinstall"
|
||||||
|
)
|
||||||
|
|
||||||
|
// testWorkerToken is the operator-supplied worker token used by the
|
||||||
|
// activation E2E tests. The fixture has no reachable control plane, but
|
||||||
|
// the worker still binds its webapp and serves /healthz, which is what
|
||||||
|
// activation verifies.
|
||||||
|
const testWorkerToken = "e2e-activation-test-token"
|
||||||
|
|
||||||
|
// activationOptions builds the work-package-4 source-install options for
|
||||||
|
// a started fixture: full activation with a fixed test token, honoring
|
||||||
|
// the RSMON_TEST_SOURCE_REPO/BRANCH overrides the staging tests use.
|
||||||
|
func activationOptions(h *Harness, f Fixture) installer.SourceInstallOptions {
|
||||||
|
opts := installer.SourceInstallOptions{
|
||||||
|
SSHOptions: installer.SSHOptions{
|
||||||
|
Host: "127.0.0.1",
|
||||||
|
Port: h.Port(),
|
||||||
|
User: f.UserOrDefault(),
|
||||||
|
IdentityFile: testKeyPath(),
|
||||||
|
KnownHostsFile: h.KnownHostsPath(),
|
||||||
|
},
|
||||||
|
Activation: installer.ActivationOptions{
|
||||||
|
Activate: true,
|
||||||
|
URL: "https://rsmon.ru",
|
||||||
|
Token: testWorkerToken,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
if repo := strings.TrimSpace(os.Getenv("RSMON_TEST_SOURCE_REPO")); repo != "" {
|
||||||
|
opts.Repo = repo
|
||||||
|
}
|
||||||
|
if branch := strings.TrimSpace(os.Getenv("RSMON_TEST_SOURCE_BRANCH")); branch != "" {
|
||||||
|
opts.Branch = branch
|
||||||
|
}
|
||||||
|
return opts
|
||||||
|
}
|
||||||
|
|
||||||
|
// fixtureByName returns the fixture entry for a fixture name.
|
||||||
|
func fixtureByName(t *testing.T, name string) Fixture {
|
||||||
|
t.Helper()
|
||||||
|
for _, f := range Fixtures() {
|
||||||
|
if f.Name == name {
|
||||||
|
return f
|
||||||
|
}
|
||||||
|
}
|
||||||
|
t.Fatalf("no fixture named %q", name)
|
||||||
|
return Fixture{}
|
||||||
|
}
|
||||||
|
|
||||||
|
// startActivatedFixture starts a fixture container and runs a full
|
||||||
|
// source install with activation against it, returning the harness, the
|
||||||
|
// options (so a test can rerun with modifications), the result, and a
|
||||||
|
// live SSH client. The caller owns client.Close and h.Stop (registered as
|
||||||
|
// a test cleanup).
|
||||||
|
func startActivatedFixture(t *testing.T, name string) (*Harness, installer.SourceInstallOptions, *installer.SourceInstallResult, *ssh.Client) {
|
||||||
|
t.Helper()
|
||||||
|
f := fixtureByName(t, name)
|
||||||
|
h, err := New("activate-"+name, f)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute)
|
||||||
|
defer cancel()
|
||||||
|
if err := h.Start(ctx); err != nil {
|
||||||
|
t.Fatalf("start %s fixture: %v", name, err)
|
||||||
|
}
|
||||||
|
t.Cleanup(func() {
|
||||||
|
if err := h.Stop(); err != nil {
|
||||||
|
t.Errorf("stop %s fixture: %v", name, err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
opts := activationOptions(h, f)
|
||||||
|
res, err := installer.SourceInstall(opts)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("activated source install on %s: %v", name, err)
|
||||||
|
}
|
||||||
|
client, err := h.Dial()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("dial %s after install: %v", name, err)
|
||||||
|
}
|
||||||
|
return h, opts, res, client
|
||||||
|
}
|
||||||
|
|
||||||
|
// countWorkerProcesses returns the number of running processes whose
|
||||||
|
// command line is exactly the installed worker binary. The anchored -f
|
||||||
|
// pattern matches both busybox and procps pgrep (busybox's -x matches the
|
||||||
|
// full argv, so -x cannot be used portably).
|
||||||
|
func countWorkerProcesses(t *testing.T, client *ssh.Client, binary string) string {
|
||||||
|
t.Helper()
|
||||||
|
out, err := RunCommand(client, "pgrep -f '^"+binary+"$' | wc -l")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("count worker processes: %v", err)
|
||||||
|
}
|
||||||
|
return strings.TrimSpace(string(out))
|
||||||
|
}
|
||||||
|
|
||||||
|
// assertActivatedService verifies the full work-package-4 remote state:
|
||||||
|
// the installed layout and permissions, the native service definition for
|
||||||
|
// the detected init, exactly one running worker process, the pid file,
|
||||||
|
// and a live /healthz (via the installed binary's own liveness
|
||||||
|
// subcommand).
|
||||||
|
func assertActivatedService(t *testing.T, client *ssh.Client, name string, res *installer.SourceInstallResult) {
|
||||||
|
t.Helper()
|
||||||
|
a := res.Activation
|
||||||
|
if a == nil {
|
||||||
|
t.Fatalf("%s: activation result missing", name)
|
||||||
|
}
|
||||||
|
|
||||||
|
out, err := RunCommand(client, "test -x "+shellQuote(a.Binary)+" && echo BIN_OK")
|
||||||
|
if err != nil || !strings.Contains(string(out), "BIN_OK") {
|
||||||
|
t.Fatalf("%s: installed binary not present/executable at %s: %q, %v", name, a.Binary, out, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
envMode, err := RunCommand(client, "stat -c '%a' "+shellQuote(a.EnvFile))
|
||||||
|
if err != nil || strings.TrimSpace(string(envMode)) != "600" {
|
||||||
|
t.Fatalf("%s: env file mode = %q, want 600 (%v)", name, envMode, err)
|
||||||
|
}
|
||||||
|
cfgMode, err := RunCommand(client, "stat -c '%a' "+shellQuote(a.ConfigDir))
|
||||||
|
if err != nil || strings.TrimSpace(string(cfgMode)) != "750" {
|
||||||
|
t.Fatalf("%s: config dir mode = %q, want 750 (%v)", name, cfgMode, err)
|
||||||
|
}
|
||||||
|
if _, err := RunCommand(client, "test -d "+shellQuote(a.DataDir)+"/webapp && echo DATA_OK"); err != nil {
|
||||||
|
t.Fatalf("%s: data dir not created at %s: %v", name, a.DataDir, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// The fixtures run no init system (sshd is PID 1), so activation
|
||||||
|
// must use the embedded supervisor; the native service definition is
|
||||||
|
// still installed for real hosts.
|
||||||
|
if a.Supervisor != "none" {
|
||||||
|
t.Fatalf("%s: supervisor = %q, want none in the fixture", name, a.Supervisor)
|
||||||
|
}
|
||||||
|
unitMode := "644"
|
||||||
|
if res.Detection.InitSystem == sshinstall.InitOpenRC {
|
||||||
|
unitMode = "755"
|
||||||
|
}
|
||||||
|
if a.UnitFile == "" {
|
||||||
|
t.Fatalf("%s: no service definition installed", name)
|
||||||
|
}
|
||||||
|
if _, err := RunCommand(client, "test -f "+shellQuote(a.UnitFile)+" && echo UNIT_OK"); err != nil {
|
||||||
|
t.Fatalf("%s: unit file missing at %s: %v", name, a.UnitFile, err)
|
||||||
|
}
|
||||||
|
gotUnitMode, err := RunCommand(client, "stat -c '%a' "+shellQuote(a.UnitFile))
|
||||||
|
if err != nil || strings.TrimSpace(string(gotUnitMode)) != unitMode {
|
||||||
|
t.Fatalf("%s: unit mode = %q, want %s (%v)", name, gotUnitMode, unitMode, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Exactly one worker process and a live /healthz.
|
||||||
|
if got := countWorkerProcesses(t, client, a.Binary); got != "1" {
|
||||||
|
t.Fatalf("%s: worker process count = %q, want 1", name, got)
|
||||||
|
}
|
||||||
|
if out, err := RunCommand(client, shellQuote(a.Binary)+" liveness"); err != nil || !strings.Contains(string(out), "ok") {
|
||||||
|
t.Fatalf("%s: installed worker /healthz check failed: %q, %v", name, out, err)
|
||||||
|
}
|
||||||
|
pidOut, err := RunCommand(client, "cat "+shellQuote(a.DataDir)+"/worker.pid")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("%s: pid file missing: %v", name, err)
|
||||||
|
}
|
||||||
|
if _, err := RunCommand(client, "kill -0 "+strings.TrimSpace(string(pidOut))+" && echo PID_OK"); err != nil {
|
||||||
|
t.Fatalf("%s: pid %s not alive: %v", name, pidOut, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// assertNoActivationLeaks verifies a rerun left no backup dir, no lock,
|
||||||
|
// no /tmp upload leftovers, and no extra worker processes.
|
||||||
|
func assertNoActivationLeaks(t *testing.T, client *ssh.Client, name string, dataDir string) {
|
||||||
|
t.Helper()
|
||||||
|
out, err := RunCommand(client,
|
||||||
|
"leak=0; [ -e "+shellQuote(dataDir)+"/.rsmon-backup ] && leak=1; "+
|
||||||
|
"[ -d "+shellQuote(dataDir)+"/.rsmon-activate.lock ] && leak=1; "+
|
||||||
|
"ls /tmp 2>/dev/null | grep -q '^rsmon-worker-act\\.' && leak=1; "+
|
||||||
|
"[ \"$leak\" -eq 1 ] && echo LEAK || echo CLEAN")
|
||||||
|
if err != nil || strings.TrimSpace(string(out)) != "CLEAN" {
|
||||||
|
t.Fatalf("%s: activation leftovers after rerun: %q, %v", name, out, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// installedBinarySHA captures the SHA-256 of the installed worker binary
|
||||||
|
// so failure tests can prove the prior binary is preserved.
|
||||||
|
func installedBinarySHA(t *testing.T, client *ssh.Client, a *installer.ActivationResult) []byte {
|
||||||
|
t.Helper()
|
||||||
|
out, err := RunCommand(client, "sha256sum "+shellQuote(a.Binary))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return bytes.TrimSpace(out)
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestSourceInstallActivationFixtures is the work-package-4 success and
|
||||||
|
// idempotency acceptance test: on every fixture a full source install
|
||||||
|
// ends with an atomically installed binary, a mode-0600 env file, the
|
||||||
|
// data dir, a service definition for the detected init, and a running
|
||||||
|
// verified worker. A rerun succeeds, keeps a single worker process,
|
||||||
|
// rewrites the env deterministically, and leaks no temp or backup files.
|
||||||
|
//
|
||||||
|
// Opt-in: set RSMON_TEST_DOCKER=1 (see make test-ssh).
|
||||||
|
func TestSourceInstallActivationFixtures(t *testing.T) {
|
||||||
|
SkipUnlessEnabled(t)
|
||||||
|
|
||||||
|
for _, f := range Fixtures() {
|
||||||
|
f := f
|
||||||
|
t.Run(f.Name, func(t *testing.T) {
|
||||||
|
_, opts, res, client := startActivatedFixture(t, f.Name)
|
||||||
|
defer client.Close() //nolint:errcheck
|
||||||
|
assertActivatedService(t, client, f.Name, res)
|
||||||
|
|
||||||
|
// Rerun: idempotent, single service, no leaks.
|
||||||
|
res2, err := installer.SourceInstall(opts)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("%s: activated rerun: %v", f.Name, err)
|
||||||
|
}
|
||||||
|
if res2.Activation == nil || res2.ResolvedCommit != res.ResolvedCommit {
|
||||||
|
t.Fatalf("%s: rerun lost activation/commit: %+v", f.Name, res2)
|
||||||
|
}
|
||||||
|
assertActivatedService(t, client, f.Name, res2)
|
||||||
|
assertNoActivationLeaks(t, client, f.Name, res2.Activation.DataDir)
|
||||||
|
|
||||||
|
envAfter, err := RunCommand(client, "cat "+shellQuote(res2.Activation.EnvFile))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("%s: read env after rerun: %v", f.Name, err)
|
||||||
|
}
|
||||||
|
if !strings.Contains(string(envAfter), "RSMON_TOKEN="+testWorkerToken) {
|
||||||
|
t.Fatalf("%s: env lost the worker token after rerun: %q", f.Name, envAfter)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestSourceInstallActivationFailureRollback is the work-package-4
|
||||||
|
// rollback acceptance test. After a successful activated install on the
|
||||||
|
// Alpine fixture, each rerun is forced to fail at a different stage -
|
||||||
|
// build (dirty checkout), activation (atomic swap sabotaged), start (the
|
||||||
|
// worker process dies at boot), and health (/healthz unreachable) - and
|
||||||
|
// every failure must leave the previous working install running with the
|
||||||
|
// same binary, the same single process, and /healthz answering.
|
||||||
|
//
|
||||||
|
// Opt-in: set RSMON_TEST_DOCKER=1 (see make test-ssh).
|
||||||
|
func TestSourceInstallActivationFailureRollback(t *testing.T) {
|
||||||
|
SkipUnlessEnabled(t)
|
||||||
|
|
||||||
|
const fixture = "alpine"
|
||||||
|
_, opts, res, client := startActivatedFixture(t, fixture)
|
||||||
|
defer client.Close() //nolint:errcheck
|
||||||
|
a := res.Activation
|
||||||
|
assertActivatedService(t, client, fixture, res)
|
||||||
|
|
||||||
|
// 1. Build failure preserves the prior install.
|
||||||
|
// A dirty tracked tree makes the rerun's checkout refuse before
|
||||||
|
// the build, so neither staging nor the service changes.
|
||||||
|
if _, err := RunCommand(client, "git -C "+shellQuote(res.Plan.BuildDir)+" checkout -q master~1 -- Makefile"); err != nil {
|
||||||
|
t.Fatalf("dirty the working tree: %v", err)
|
||||||
|
}
|
||||||
|
before := installedBinarySHA(t, client, a)
|
||||||
|
_, err := installer.SourceInstall(opts)
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "check out branch") {
|
||||||
|
t.Fatalf("rerun on dirty tree err = %v, want checkout failure", err)
|
||||||
|
}
|
||||||
|
assertActivatedService(t, client, fixture, res)
|
||||||
|
if !bytes.Equal(before, installedBinarySHA(t, client, a)) {
|
||||||
|
t.Fatalf("installed binary changed after build-failure rerun")
|
||||||
|
}
|
||||||
|
// Restore the tree so later reruns can check out again.
|
||||||
|
if _, err := RunCommand(client, "git -C "+shellQuote(res.Plan.BuildDir)+" checkout -q HEAD -- Makefile"); err != nil {
|
||||||
|
t.Fatalf("restore working tree: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// 2. Activation failure preserves the prior install.
|
||||||
|
// Pre-creating the atomic-swap temp path as a directory makes the
|
||||||
|
// binary install step fail mid-activation; the rollback trap must
|
||||||
|
// restore the previous binary/env and restart the prior service.
|
||||||
|
if _, err := RunCommand(client, "mkdir -p /usr/local/bin/rsmon-worker.new"); err != nil {
|
||||||
|
t.Fatalf("sabotage the atomic binary swap: %v", err)
|
||||||
|
}
|
||||||
|
_, err = installer.SourceInstall(opts)
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "activate worker service") {
|
||||||
|
t.Fatalf("activation-failure rerun err = %v, want activate failure", err)
|
||||||
|
}
|
||||||
|
assertActivatedService(t, client, fixture, res)
|
||||||
|
if !bytes.Equal(before, installedBinarySHA(t, client, a)) {
|
||||||
|
t.Fatalf("installed binary changed after activation-failure rerun")
|
||||||
|
}
|
||||||
|
assertNoActivationLeaks(t, client, fixture, a.DataDir)
|
||||||
|
if _, err := RunCommand(client, "rm -rf /usr/local/bin/rsmon-worker.new"); err != nil {
|
||||||
|
t.Fatalf("clean the swap sabotage: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// 3. Start failure preserves the prior install.
|
||||||
|
// WORKER_CLUSTER_ENABLED=true without WORKER_LOGIN/PASSWORD makes
|
||||||
|
// the new worker exit at boot (cluster init is fatal), so the
|
||||||
|
// activation supervisor detects the process died and rolls back.
|
||||||
|
envPath := filepath.Join(t.TempDir(), "worker.env")
|
||||||
|
startEnv := "RSMON_URL=https://rsmon.ru\nRSMON_TOKEN=" + testWorkerToken + "\nWORKER_CLUSTER_ENABLED=true\n"
|
||||||
|
if err := os.WriteFile(envPath, []byte(startEnv), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
startOpts := opts
|
||||||
|
startOpts.Activation.EnvFile = envPath
|
||||||
|
_, err = installer.SourceInstall(startOpts)
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "start failure") {
|
||||||
|
t.Fatalf("start-failure rerun err = %v, want start failure", err)
|
||||||
|
}
|
||||||
|
assertActivatedService(t, client, fixture, res)
|
||||||
|
if !bytes.Equal(before, installedBinarySHA(t, client, a)) {
|
||||||
|
t.Fatalf("installed binary changed after start-failure rerun")
|
||||||
|
}
|
||||||
|
assertNoActivationLeaks(t, client, fixture, a.DataDir)
|
||||||
|
|
||||||
|
// 4. Health failure preserves the prior install.
|
||||||
|
// WORKER_HOST=255.255.255.255 makes the new worker bind fail and
|
||||||
|
// /healthz unreachable; the process stays up but the health gate
|
||||||
|
// fails and the rollback restores the prior loopback deployment.
|
||||||
|
healthOpts := opts
|
||||||
|
healthOpts.Activation.Host = "255.255.255.255"
|
||||||
|
_, err = installer.SourceInstall(healthOpts)
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "health failure") {
|
||||||
|
t.Fatalf("health-failure rerun err = %v, want health failure", err)
|
||||||
|
}
|
||||||
|
assertActivatedService(t, client, fixture, res)
|
||||||
|
if !bytes.Equal(before, installedBinarySHA(t, client, a)) {
|
||||||
|
t.Fatalf("installed binary changed after health-failure rerun")
|
||||||
|
}
|
||||||
|
assertNoActivationLeaks(t, client, fixture, a.DataDir)
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestSourceInstallActivationNoStart installs the full layout (binary,
|
||||||
|
// env, data dir, service definition) without starting the worker, and
|
||||||
|
// asserts nothing runs afterward. Reruns stay idempotent.
|
||||||
|
//
|
||||||
|
// Opt-in: set RSMON_TEST_DOCKER=1 (see make test-ssh).
|
||||||
|
func TestSourceInstallActivationNoStart(t *testing.T) {
|
||||||
|
SkipUnlessEnabled(t)
|
||||||
|
|
||||||
|
const fixture = "alpine"
|
||||||
|
f := fixtureByName(t, fixture)
|
||||||
|
h, err := New("nostart-"+fixture, f)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute)
|
||||||
|
defer cancel()
|
||||||
|
if err := h.Start(ctx); err != nil {
|
||||||
|
t.Fatalf("start %s fixture: %v", fixture, err)
|
||||||
|
}
|
||||||
|
t.Cleanup(func() {
|
||||||
|
if err := h.Stop(); err != nil {
|
||||||
|
t.Errorf("stop: %v", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
opts := activationOptions(h, f)
|
||||||
|
opts.Activation.NoStart = true
|
||||||
|
res, err := installer.SourceInstall(opts)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("no-start install: %v", err)
|
||||||
|
}
|
||||||
|
client, err := h.Dial()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("dial: %v", err)
|
||||||
|
}
|
||||||
|
defer client.Close() //nolint:errcheck
|
||||||
|
a := res.Activation
|
||||||
|
if a == nil || a.Started {
|
||||||
|
t.Fatalf("no-start activation result = %+v", a)
|
||||||
|
}
|
||||||
|
if out, err := RunCommand(client, "test -x "+shellQuote(a.Binary)+" && test -f "+shellQuote(a.EnvFile)+" && echo INSTALLED"); err != nil || !strings.Contains(string(out), "INSTALLED") {
|
||||||
|
t.Fatalf("no-start did not install the layout: %q, %v", out, err)
|
||||||
|
}
|
||||||
|
if got := countWorkerProcesses(t, client, a.Binary); got != "0" {
|
||||||
|
t.Fatalf("no-start left a running worker: %q", got)
|
||||||
|
}
|
||||||
|
// A rerun is idempotent and still starts nothing.
|
||||||
|
if _, err := installer.SourceInstall(opts); err != nil {
|
||||||
|
t.Fatalf("no-start rerun: %v", err)
|
||||||
|
}
|
||||||
|
if got := countWorkerProcesses(t, client, a.Binary); got != "0" {
|
||||||
|
t.Fatalf("no-start rerun left a running worker: %q", got)
|
||||||
|
}
|
||||||
|
assertNoActivationLeaks(t, client, fixture, a.DataDir)
|
||||||
|
}
|
||||||
@@ -325,6 +325,9 @@ func TestSourceInstallFixtures(t *testing.T) {
|
|||||||
IdentityFile: testKeyPath(),
|
IdentityFile: testKeyPath(),
|
||||||
KnownHostsFile: h.KnownHostsPath(),
|
KnownHostsFile: h.KnownHostsPath(),
|
||||||
},
|
},
|
||||||
|
// Staging-only: service activation (work package 4) is
|
||||||
|
// exercised by the dedicated activation tests.
|
||||||
|
Activation: installer.ActivationOptions{Activate: false},
|
||||||
}
|
}
|
||||||
if repo := strings.TrimSpace(os.Getenv("RSMON_TEST_SOURCE_REPO")); repo != "" {
|
if repo := strings.TrimSpace(os.Getenv("RSMON_TEST_SOURCE_REPO")); repo != "" {
|
||||||
opts.Repo = repo
|
opts.Repo = repo
|
||||||
@@ -446,6 +449,7 @@ func TestSourceInstallDirtyCheckoutPreservesStaging(t *testing.T) {
|
|||||||
IdentityFile: testKeyPath(),
|
IdentityFile: testKeyPath(),
|
||||||
KnownHostsFile: h.KnownHostsPath(),
|
KnownHostsFile: h.KnownHostsPath(),
|
||||||
},
|
},
|
||||||
|
Activation: installer.ActivationOptions{Activate: false},
|
||||||
}
|
}
|
||||||
if repo := strings.TrimSpace(os.Getenv("RSMON_TEST_SOURCE_REPO")); repo != "" {
|
if repo := strings.TrimSpace(os.Getenv("RSMON_TEST_SOURCE_REPO")); repo != "" {
|
||||||
opts.Repo = repo
|
opts.Repo = repo
|
||||||
|
|||||||
@@ -478,6 +478,16 @@ func ValidateEnvironmentFile(path string) error {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("read worker environment: %w", err)
|
return fmt.Errorf("read worker environment: %w", err)
|
||||||
}
|
}
|
||||||
|
_, err = parseEnvironmentContent(data)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// parseEnvironmentContent validates systemd-safe KEY=VALUE environment
|
||||||
|
// content and returns the parsed map. Values must not use quoting,
|
||||||
|
// interpolation, or whitespace; RSMON_URL and RSMON_TOKEN are required and
|
||||||
|
// must not be duplicated. Operating on the already-read bytes (rather
|
||||||
|
// than a path) keeps callers free of read-to-validate TOCTOU races.
|
||||||
|
func parseEnvironmentContent(data []byte) (map[string]string, error) {
|
||||||
values := make(map[string]string)
|
values := make(map[string]string)
|
||||||
seenRequired := make(map[string]bool)
|
seenRequired := make(map[string]bool)
|
||||||
for number, line := range strings.Split(string(data), "\n") {
|
for number, line := range strings.Split(string(data), "\n") {
|
||||||
@@ -486,30 +496,30 @@ func ValidateEnvironmentFile(path string) error {
|
|||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
if strings.ContainsRune(line, '\r') {
|
if strings.ContainsRune(line, '\r') {
|
||||||
return fmt.Errorf("worker environment line %d contains a carriage return", lineNumber)
|
return nil, fmt.Errorf("worker environment line %d contains a carriage return", lineNumber)
|
||||||
}
|
}
|
||||||
key, value, ok := strings.Cut(line, "=")
|
key, value, ok := strings.Cut(line, "=")
|
||||||
if !ok || !envKeyPattern.MatchString(key) {
|
if !ok || !envKeyPattern.MatchString(key) {
|
||||||
return fmt.Errorf("worker environment line %d must use KEY=VALUE syntax", lineNumber)
|
return nil, fmt.Errorf("worker environment line %d must use KEY=VALUE syntax", lineNumber)
|
||||||
}
|
}
|
||||||
if err := validateEnvValue(key, value); err != nil {
|
if err := validateEnvValue(key, value); err != nil {
|
||||||
return fmt.Errorf("worker environment line %d: %w", lineNumber, err)
|
return nil, fmt.Errorf("worker environment line %d: %w", lineNumber, err)
|
||||||
}
|
}
|
||||||
if key == "RSMON_URL" || key == "RSMON_TOKEN" {
|
if key == "RSMON_URL" || key == "RSMON_TOKEN" {
|
||||||
if seenRequired[key] {
|
if seenRequired[key] {
|
||||||
return fmt.Errorf("worker environment line %d duplicates %s", lineNumber, key)
|
return nil, fmt.Errorf("worker environment line %d duplicates %s", lineNumber, key)
|
||||||
}
|
}
|
||||||
seenRequired[key] = true
|
seenRequired[key] = true
|
||||||
}
|
}
|
||||||
values[key] = value
|
values[key] = value
|
||||||
}
|
}
|
||||||
if err := ValidateURL(values["RSMON_URL"]); err != nil {
|
if err := ValidateURL(values["RSMON_URL"]); err != nil {
|
||||||
return err
|
return nil, err
|
||||||
}
|
}
|
||||||
if err := ValidateToken(values["RSMON_TOKEN"]); err != nil {
|
if err := ValidateToken(values["RSMON_TOKEN"]); err != nil {
|
||||||
return err
|
return nil, err
|
||||||
}
|
}
|
||||||
return nil
|
return values, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// validateEnvValue enforces the value rules shared by the strict env
|
// validateEnvValue enforces the value rules shared by the strict env
|
||||||
|
|||||||
769
internal/installer/sourceactivate.go
Обычный файл
769
internal/installer/sourceactivate.go
Обычный файл
@@ -0,0 +1,769 @@
|
|||||||
|
package installer
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"rocketgit.ru/rsmon/worker/internal/sshinstall"
|
||||||
|
)
|
||||||
|
|
||||||
|
// ActivationOptions drives work package 4 of docs/source-installation.md:
|
||||||
|
// the atomic activation of the staged worker binary. It reuses the
|
||||||
|
// classic installer's environment resolution (resolveInstallEnv /
|
||||||
|
// renderEnvFile), on-disk layout (resolvePaths), and hardened systemd
|
||||||
|
// unit renderer, then installs the binary, env file, data dir, and the
|
||||||
|
// detected init's service definition atomically, starts/restarts the
|
||||||
|
// worker, and verifies the process and /healthz before declaring the
|
||||||
|
// activation a success. Any activation/start/health failure rolls back to
|
||||||
|
// the previous working install.
|
||||||
|
type ActivationOptions struct {
|
||||||
|
// Activate performs the atomic install + service activation after the
|
||||||
|
// staging build. When false SourceInstall stops at the staging build
|
||||||
|
// (the work-package-3 boundary) and touches no service configuration.
|
||||||
|
Activate bool
|
||||||
|
|
||||||
|
// Name is the instance name ("" = the primary rsmon-worker service).
|
||||||
|
// A named instance gets rsmon-worker-<name> paths and its own unit,
|
||||||
|
// matching the classic installer's multi-instance layout.
|
||||||
|
Name string
|
||||||
|
// URL is RSMON_URL (default https://rsmon.ru).
|
||||||
|
URL string
|
||||||
|
// Token is the worker API token (RSMON_TOKEN). Required for
|
||||||
|
// activation; prefer supplying it through a file at the CLI layer so
|
||||||
|
// it never appears in the process list.
|
||||||
|
Token string
|
||||||
|
// PublicURL is the advertised public origin (PUBLIC_URL).
|
||||||
|
PublicURL string
|
||||||
|
// Host is the worker webapp bind address (WORKER_HOST; default
|
||||||
|
// 127.0.0.1 as resolved by the classic installer).
|
||||||
|
Host string
|
||||||
|
// Port is the worker webapp bind port (WORKER_PORT; default 27401,
|
||||||
|
// required for named instances).
|
||||||
|
Port string
|
||||||
|
// Login / Password are the operator-console basic-auth credentials
|
||||||
|
// (WORKER_LOGIN / WORKER_PASSWORD). Both must be set or both empty.
|
||||||
|
Login string
|
||||||
|
Password string
|
||||||
|
// EnvFile is a local systemd-safe env file uploaded instead of the
|
||||||
|
// individual knobs. Validated with the classic strict parser.
|
||||||
|
EnvFile string
|
||||||
|
// NoStart installs the binary, env, data dir, and service definition
|
||||||
|
// without starting or restarting the worker.
|
||||||
|
NoStart bool
|
||||||
|
}
|
||||||
|
|
||||||
|
// ActivationResult records where and how the staged worker was activated.
|
||||||
|
type ActivationResult struct {
|
||||||
|
Binary string // installed binary path
|
||||||
|
ConfigDir string
|
||||||
|
EnvFile string
|
||||||
|
DataDir string
|
||||||
|
UnitFile string // empty when no service definition was installed
|
||||||
|
UnitName string // systemd unit / rc-service name
|
||||||
|
Supervisor string // "systemd", "openrc", or "supervisor" (embedded fallback)
|
||||||
|
Started bool
|
||||||
|
}
|
||||||
|
|
||||||
|
// installOptions projects the activation knobs onto the classic
|
||||||
|
// installer's option shape so the shared env resolution is reused.
|
||||||
|
func (a ActivationOptions) installOptions() InstallOptions {
|
||||||
|
return InstallOptions{
|
||||||
|
URL: a.URL,
|
||||||
|
Token: a.Token,
|
||||||
|
PublicURL: a.PublicURL,
|
||||||
|
Host: a.Host,
|
||||||
|
Port: a.Port,
|
||||||
|
Login: a.Login,
|
||||||
|
Password: a.Password,
|
||||||
|
Name: strings.TrimSpace(a.Name),
|
||||||
|
NoStart: a.NoStart,
|
||||||
|
EnvFile: a.EnvFile,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// renderEnv resolves the full worker environment exactly as the classic
|
||||||
|
// installer would and renders the canonical systemd-safe env file. The
|
||||||
|
// env file (when configured) is read exactly once and validated from the
|
||||||
|
// in-memory bytes, so a hostile local writer cannot swap the file
|
||||||
|
// between the read used for validation and the read used for the render.
|
||||||
|
func (a ActivationOptions) renderEnv() ([]byte, error) {
|
||||||
|
name := strings.TrimSpace(a.Name)
|
||||||
|
if err := validateInstanceName(name); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
var fileEnv map[string]string
|
||||||
|
if a.EnvFile != "" {
|
||||||
|
data, err := os.ReadFile(a.EnvFile)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("--env-file: %w", err)
|
||||||
|
}
|
||||||
|
fe, perr := parseEnvironmentContent(data)
|
||||||
|
if perr != nil {
|
||||||
|
return nil, fmt.Errorf("--env-file: %w", perr)
|
||||||
|
}
|
||||||
|
fileEnv = fe
|
||||||
|
}
|
||||||
|
values, err := resolveInstallEnv(a.installOptions(), name, fileEnv)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return renderEnvFile(values), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// unitContent renders the service definition for the detected init
|
||||||
|
// system. systemd gets the classic hardened unit; OpenRC gets a native
|
||||||
|
// init script. An unknown init produces no unit (the no-service gate:
|
||||||
|
// the embedded supervisor still installs and manages the worker, but no
|
||||||
|
// native service definition is written).
|
||||||
|
func unitContent(init sshinstall.InitSystem, p paths) (content string, file string, mode string) {
|
||||||
|
svcName := strings.TrimSuffix(p.unitName, ".service")
|
||||||
|
switch init {
|
||||||
|
case sshinstall.InitSystemd:
|
||||||
|
return systemdUnitFor(p), p.unitFile, "0644"
|
||||||
|
case sshinstall.InitOpenRC:
|
||||||
|
return openrcInitFor(p), "/etc/init.d/" + svcName, "0755"
|
||||||
|
default:
|
||||||
|
return "", "", ""
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// runningInitScript reports the init system that is *actually running*
|
||||||
|
// (not merely installed): systemd, openrc, or none. The installer writes
|
||||||
|
// the service definition for the detected init but drives start/restart
|
||||||
|
// through whichever supervisor is usable right now; a container or chroot
|
||||||
|
// where no init runs falls back to the embedded supervisor.
|
||||||
|
const runningInitScript = `set -eu
|
||||||
|
if [ -d /run/systemd/system ] && command -v systemctl >/dev/null 2>&1; then
|
||||||
|
echo systemd
|
||||||
|
elif [ -e /run/openrc/softlevel ] && command -v rc-service >/dev/null 2>&1; then
|
||||||
|
echo openrc
|
||||||
|
else
|
||||||
|
echo none
|
||||||
|
fi`
|
||||||
|
|
||||||
|
// normalizeSupervisor bounds the running-init probe output to the three
|
||||||
|
// supervisor kinds the activation script understands.
|
||||||
|
func normalizeSupervisor(raw string) string {
|
||||||
|
switch strings.TrimSpace(raw) {
|
||||||
|
case "systemd", "openrc":
|
||||||
|
return strings.TrimSpace(raw)
|
||||||
|
default:
|
||||||
|
return "none"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// activateScript renders the single idempotent remote activation script.
|
||||||
|
// It is structured as a transaction over the previous install:
|
||||||
|
//
|
||||||
|
// 1. acquire an activation lock so concurrent runs cannot mutate the same
|
||||||
|
// deployment, and recover any interrupted prior activation from its
|
||||||
|
// leftover backup marker;
|
||||||
|
// 2. validate the staged binary (fails before any state is touched);
|
||||||
|
// 3. snapshot the prior binary/env/unit (and the unit's enable state)
|
||||||
|
// into a backup dir and write a recovery marker;
|
||||||
|
// 4. arm a rollback EXIT/HUP/INT/TERM trap that restores the snapshot,
|
||||||
|
// restores the prior enable state (or disables a freshly-installed
|
||||||
|
// unit on a fresh failure), and restarts the prior service;
|
||||||
|
// 5. install the new binary, env (mode 0600), data dir, and service
|
||||||
|
// definition atomically (temp file + rename);
|
||||||
|
// 6. start/restart through the active supervisor and verify the service
|
||||||
|
// stays active (systemd `is-active` / rc-service `status` / the
|
||||||
|
// embedded supervisor's zombie-aware pid check) and answers /healthz;
|
||||||
|
// 7. on success, drop the backup and release the lock; on failure the
|
||||||
|
// trap restores the prior install and the script exits non-zero.
|
||||||
|
//
|
||||||
|
// Reruns are idempotent: restart always stops the previous instance
|
||||||
|
// first, so exactly one worker process exists, and every temp file
|
||||||
|
// (backup dir, .new files, uploaded env/unit, lock) is removed on success
|
||||||
|
// and failure.
|
||||||
|
func activateScript(p activateParams) string {
|
||||||
|
r := strings.NewReplacer(
|
||||||
|
"@@STAGE@@", shellQuote(p.Stage),
|
||||||
|
"@@BINARY@@", shellQuote(p.Binary),
|
||||||
|
"@@CONFIG_DIR@@", shellQuote(p.ConfigDir),
|
||||||
|
"@@ENV_FILE@@", shellQuote(p.EnvFile),
|
||||||
|
"@@ENV_TMP@@", shellQuote(p.EnvTmp),
|
||||||
|
"@@DATA_DIR@@", shellQuote(p.DataDir),
|
||||||
|
"@@UNIT_TMP@@", shellQuote(p.UnitTmp),
|
||||||
|
"@@UNIT_FILE@@", shellQuote(p.UnitFile),
|
||||||
|
"@@UNIT_MODE@@", p.UnitMode,
|
||||||
|
"@@UNIT_NAME@@", shellQuote(p.UnitName),
|
||||||
|
"@@RC_NAME@@", shellQuote(p.RCName),
|
||||||
|
"@@SUPERVISOR@@", p.Supervisor,
|
||||||
|
"@@NO_START@@", p.NoStart,
|
||||||
|
)
|
||||||
|
return r.Replace(activateTemplate)
|
||||||
|
}
|
||||||
|
|
||||||
|
// activateTemplate is the remote script body. Sentinels (@@..@@) are
|
||||||
|
// substituted by activateScript; every operator-controlled value is
|
||||||
|
// single-quoted and validated on the controller side first.
|
||||||
|
const activateTemplate = `set -eu
|
||||||
|
|
||||||
|
stage=@@STAGE@@
|
||||||
|
binary=@@BINARY@@
|
||||||
|
config_dir=@@CONFIG_DIR@@
|
||||||
|
env_file=@@ENV_FILE@@
|
||||||
|
env_tmp=@@ENV_TMP@@
|
||||||
|
data_dir=@@DATA_DIR@@
|
||||||
|
unit_tmp=@@UNIT_TMP@@
|
||||||
|
unit_file=@@UNIT_FILE@@
|
||||||
|
unit_mode=@@UNIT_MODE@@
|
||||||
|
unit_name=@@UNIT_NAME@@
|
||||||
|
rc_name=@@RC_NAME@@
|
||||||
|
supervisor=@@SUPERVISOR@@
|
||||||
|
no_start=@@NO_START@@
|
||||||
|
|
||||||
|
backup_dir="$data_dir/.rsmon-backup"
|
||||||
|
lock_dir="$data_dir/.rsmon-activate.lock"
|
||||||
|
pid_file="$data_dir/worker.pid"
|
||||||
|
log_file="$data_dir/worker.log"
|
||||||
|
|
||||||
|
run_env() {
|
||||||
|
# Execute "$@" with a fresh environment built only from the env file
|
||||||
|
# plus PATH, HOME, and the webapp data dir. Sourcing the env file in a
|
||||||
|
# clean env -i shell prevents variables from a previous activation
|
||||||
|
# (for example WORKER_CLUSTER_ENABLED) from leaking into the new
|
||||||
|
# worker process after a rollback restores an older env file.
|
||||||
|
env -i PATH="/usr/bin:/bin:/sbin:/usr/sbin" HOME="$data_dir" \
|
||||||
|
RSMON_WEBAPP_DATA_DIR="$data_dir/webapp" \
|
||||||
|
RSMON_WORKER_ENV_FILE="$env_file" sh -c '
|
||||||
|
set -a
|
||||||
|
. "$RSMON_WORKER_ENV_FILE"
|
||||||
|
set +a
|
||||||
|
unset RSMON_WORKER_ENV_FILE
|
||||||
|
exec "$@"
|
||||||
|
' sh "$@"
|
||||||
|
}
|
||||||
|
|
||||||
|
process_up() {
|
||||||
|
# True when the given pid is a live, non-zombie process. A zombie
|
||||||
|
# still answers kill -0 (its task struct exists until reaped), and in
|
||||||
|
# a container where PID 1 does not reap children a killed worker can
|
||||||
|
# linger as a zombie for a long time, so the /proc state must be
|
||||||
|
# checked explicitly.
|
||||||
|
[ -n "$1" ] || return 1
|
||||||
|
kill -0 "$1" 2>/dev/null || return 1
|
||||||
|
state="$(awk '{print $3}' "/proc/$1/stat" 2>/dev/null || true)"
|
||||||
|
[ "$state" != "Z" ] || return 1
|
||||||
|
}
|
||||||
|
|
||||||
|
process_is_worker() {
|
||||||
|
# True when the given pid's executable is (or was, before an atomic
|
||||||
|
# binary swap) the configured worker binary, so a stale or recycled
|
||||||
|
# pid file can never make the supervisor kill an unrelated process.
|
||||||
|
[ -n "$1" ] || return 1
|
||||||
|
exe="$(readlink "/proc/$1/exe" 2>/dev/null || true)"
|
||||||
|
case "$exe" in
|
||||||
|
"$binary"|"$binary (deleted)") return 0 ;;
|
||||||
|
esac
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
|
||||||
|
stop_process() {
|
||||||
|
if [ -f "$pid_file" ]; then
|
||||||
|
pid="$(cat "$pid_file" 2>/dev/null || true)"
|
||||||
|
if [ -n "$pid" ] && process_is_worker "$pid" && process_up "$pid"; then
|
||||||
|
kill "$pid" 2>/dev/null || true
|
||||||
|
i=0
|
||||||
|
while [ "$i" -lt 10 ]; do
|
||||||
|
if ! process_up "$pid"; then
|
||||||
|
break
|
||||||
|
fi
|
||||||
|
sleep 1
|
||||||
|
i=$((i + 1))
|
||||||
|
done
|
||||||
|
kill -9 "$pid" 2>/dev/null || true
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
rm -f "$pid_file"
|
||||||
|
}
|
||||||
|
|
||||||
|
start_process() {
|
||||||
|
stop_process
|
||||||
|
mkdir -p "$data_dir"
|
||||||
|
run_env sh -c 'nohup "$1" >>"$2" 2>&1 & echo $! > "$3"' sh "$binary" "$log_file" "$pid_file"
|
||||||
|
}
|
||||||
|
|
||||||
|
process_alive() {
|
||||||
|
[ -f "$pid_file" ] || return 1
|
||||||
|
pid="$(cat "$pid_file" 2>/dev/null || true)"
|
||||||
|
process_is_worker "$pid" || return 1
|
||||||
|
process_up "$pid"
|
||||||
|
}
|
||||||
|
|
||||||
|
worker_come_up() {
|
||||||
|
# Wait up to 10s for the freshly-started worker to become a live,
|
||||||
|
# non-zombie process running the configured binary. The recorded pid
|
||||||
|
# starts as the nohup/sh child before it execs the worker, so early
|
||||||
|
# readlink checks can transiently see the interpreter; this retry both
|
||||||
|
# tolerates that exec window and catches genuine immediate deaths.
|
||||||
|
i=0
|
||||||
|
while [ "$i" -lt 10 ]; do
|
||||||
|
if process_alive; then
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
sleep 1
|
||||||
|
i=$((i + 1))
|
||||||
|
done
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
|
||||||
|
health_ok() {
|
||||||
|
run_env "$binary" liveness >/dev/null 2>&1
|
||||||
|
}
|
||||||
|
|
||||||
|
svc_active() {
|
||||||
|
# The health loop verifies the service is active through the
|
||||||
|
# supervisor (never through a pid file for the init-managed paths).
|
||||||
|
case "$supervisor" in
|
||||||
|
systemd) systemctl is-active --quiet "$unit_name" ;;
|
||||||
|
openrc) rc-service "$rc_name" status >/dev/null 2>&1 ;;
|
||||||
|
none) process_alive ;;
|
||||||
|
esac
|
||||||
|
}
|
||||||
|
|
||||||
|
stop_svc() {
|
||||||
|
case "$supervisor" in
|
||||||
|
systemd) systemctl stop "$unit_name" >/dev/null 2>&1 || true ;;
|
||||||
|
openrc) rc-service "$rc_name" stop >/dev/null 2>&1 || true ;;
|
||||||
|
none) stop_process ;;
|
||||||
|
esac
|
||||||
|
}
|
||||||
|
|
||||||
|
restart_svc() {
|
||||||
|
case "$supervisor" in
|
||||||
|
systemd)
|
||||||
|
systemctl daemon-reload >/dev/null 2>&1
|
||||||
|
if ! systemctl restart "$unit_name" >/dev/null 2>&1; then
|
||||||
|
echo "start failure: systemctl restart $unit_name failed" >&2
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
if ! systemctl is-active --quiet "$unit_name"; then
|
||||||
|
echo "start failure: unit $unit_name is not active" >&2
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
;;
|
||||||
|
openrc)
|
||||||
|
if ! rc-service "$rc_name" restart >/dev/null 2>&1; then
|
||||||
|
echo "start failure: rc-service restart $rc_name failed" >&2
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
if ! rc-service "$rc_name" status >/dev/null 2>&1; then
|
||||||
|
echo "start failure: service $rc_name is not running" >&2
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
;;
|
||||||
|
none)
|
||||||
|
start_process
|
||||||
|
if ! worker_come_up; then
|
||||||
|
echo "start failure: worker process did not stay alive" >&2
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
i=0
|
||||||
|
while [ "$i" -lt 30 ]; do
|
||||||
|
if health_ok; then
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
# A worker that stops while the health gate is still probing is a
|
||||||
|
# start failure (it never became a stable service), not merely an
|
||||||
|
# unhealthy-but-running one.
|
||||||
|
if ! svc_active; then
|
||||||
|
echo "start failure: worker service is not active" >&2
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
sleep 1
|
||||||
|
i=$((i + 1))
|
||||||
|
done
|
||||||
|
echo "health failure: worker did not answer /healthz" >&2
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
|
||||||
|
# --- unit enable helpers -------------------------------------------------
|
||||||
|
# unit_was_enabled records whether the unit referenced by the current
|
||||||
|
# params is enabled (best-effort; a missing init leaves it disabled).
|
||||||
|
unit_was_enabled() {
|
||||||
|
case "$supervisor" in
|
||||||
|
systemd) systemctl is-enabled "$unit_name" >/dev/null 2>&1 && unit_enabled=1 ;;
|
||||||
|
openrc) [ -e "/etc/runlevels/default/$rc_name" ] && unit_enabled=1 ;;
|
||||||
|
esac
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
# set_unit_enabled "$1" enables (1) or disables (0) the unit.
|
||||||
|
set_unit_enabled() {
|
||||||
|
case "$supervisor" in
|
||||||
|
systemd)
|
||||||
|
systemctl daemon-reload >/dev/null 2>&1 || true
|
||||||
|
if [ "$1" -eq 1 ]; then
|
||||||
|
systemctl enable "$unit_name" >/dev/null 2>&1 || true
|
||||||
|
else
|
||||||
|
systemctl disable "$unit_name" >/dev/null 2>&1 || true
|
||||||
|
fi
|
||||||
|
;;
|
||||||
|
openrc)
|
||||||
|
if [ "$1" -eq 1 ]; then
|
||||||
|
rc-update add "$rc_name" default >/dev/null 2>&1 || true
|
||||||
|
else
|
||||||
|
rc-update del "$rc_name" default >/dev/null 2>&1 || true
|
||||||
|
fi
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
# --- activation lock ------------------------------------------------------
|
||||||
|
# Prevent concurrent activations on the same host from racing the
|
||||||
|
# deployed-state mutation. A stale lock (left over from a SIGKILL'd run
|
||||||
|
# whose pid is no longer alive) is broken automatically.
|
||||||
|
acquire_lock() {
|
||||||
|
mkdir -p "$data_dir"
|
||||||
|
if [ -d "$lock_dir" ]; then
|
||||||
|
lockpid="$(cat "$lock_dir/pid" 2>/dev/null || true)"
|
||||||
|
if [ -n "$lockpid" ] && ! kill -0 "$lockpid" 2>/dev/null; then
|
||||||
|
rm -rf "$lock_dir" 2>/dev/null || true
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
i=0
|
||||||
|
while [ "$i" -lt 60 ]; do
|
||||||
|
if mkdir "$lock_dir" 2>/dev/null; then
|
||||||
|
chmod 0700 "$lock_dir" 2>/dev/null || true
|
||||||
|
echo $$ > "$lock_dir/pid" 2>/dev/null || true
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
sleep 1
|
||||||
|
i=$((i + 1))
|
||||||
|
done
|
||||||
|
echo "another rsmon-worker activation is in progress ($lock_dir)" >&2
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
release_lock() {
|
||||||
|
rm -rf "$lock_dir" 2>/dev/null || true
|
||||||
|
}
|
||||||
|
|
||||||
|
if ! acquire_lock; then
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
# Any early failure (before the rollback trap is armed) still releases the
|
||||||
|
# lock.
|
||||||
|
trap 'release_lock' EXIT
|
||||||
|
|
||||||
|
# --- interrupted-run recovery --------------------------------------------
|
||||||
|
# A previous activation that was killed mid-flight (SSH drop, SIGKILL)
|
||||||
|
# leaves its backup marker behind. Restore that snapshot before the fresh
|
||||||
|
# activation runs, so the host never stays half-activated and the next run
|
||||||
|
# starts from a consistent prior state.
|
||||||
|
unit_enabled=0
|
||||||
|
recover_backup() {
|
||||||
|
if [ ! -f "$backup_dir/.marker" ]; then
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
echo "recovering interrupted activation from $backup_dir" >&2
|
||||||
|
if [ -f "$backup_dir/binary" ]; then
|
||||||
|
mkdir -p "$(dirname "$binary")" || true
|
||||||
|
cp -p "$backup_dir/binary" "$binary" || true
|
||||||
|
fi
|
||||||
|
if [ -f "$backup_dir/worker.env" ]; then
|
||||||
|
mkdir -p "$config_dir" || true
|
||||||
|
cp -p "$backup_dir/worker.env" "$env_file" || true
|
||||||
|
fi
|
||||||
|
if [ -n "$unit_file" ] && [ -f "$backup_dir/unit" ]; then
|
||||||
|
mkdir -p "$(dirname "$unit_file")" || true
|
||||||
|
cp -p "$backup_dir/unit" "$unit_file" || true
|
||||||
|
chmod "$unit_mode" "$unit_file" || true
|
||||||
|
fi
|
||||||
|
if [ "$(cat "$backup_dir/.marker" 2>/dev/null || true)" = "1" ]; then
|
||||||
|
unit_enabled=1
|
||||||
|
fi
|
||||||
|
set_unit_enabled "$unit_enabled" || true
|
||||||
|
if [ -f "$backup_dir/binary" ] && [ -f "$backup_dir/worker.env" ] && [ "$no_start" -ne 1 ]; then
|
||||||
|
restart_svc || true
|
||||||
|
fi
|
||||||
|
rm -rf "$backup_dir" || true
|
||||||
|
}
|
||||||
|
recover_backup
|
||||||
|
|
||||||
|
# --- staged binary validation (no state touched) -------------------------
|
||||||
|
# A corrupt or missing staging build aborts here and leaves the prior
|
||||||
|
# install completely untouched (the rollback trap is not armed yet).
|
||||||
|
"$stage" --version >/dev/null
|
||||||
|
|
||||||
|
# --- snapshot the prior install ------------------------------------------
|
||||||
|
rm -rf "$backup_dir"
|
||||||
|
mkdir -p "$backup_dir"
|
||||||
|
had_binary=0
|
||||||
|
had_env=0
|
||||||
|
had_unit=0
|
||||||
|
unit_enabled=0
|
||||||
|
if [ -e "$binary" ]; then
|
||||||
|
cp -p "$binary" "$backup_dir/binary"
|
||||||
|
had_binary=1
|
||||||
|
fi
|
||||||
|
if [ -e "$env_file" ]; then
|
||||||
|
cp -p "$env_file" "$backup_dir/worker.env"
|
||||||
|
had_env=1
|
||||||
|
fi
|
||||||
|
if [ -n "$unit_file" ] && [ -e "$unit_file" ]; then
|
||||||
|
had_unit=1
|
||||||
|
cp -p "$unit_file" "$backup_dir/unit"
|
||||||
|
unit_was_enabled
|
||||||
|
fi
|
||||||
|
# The marker is written only after the snapshot copies so recovery never
|
||||||
|
# sees a partial snapshot; it records whether the prior unit was enabled.
|
||||||
|
printf '%s\n' "$unit_enabled" > "$backup_dir/.marker"
|
||||||
|
|
||||||
|
# --- rollback trap --------------------------------------------------------
|
||||||
|
# Any failure from here on restores the snapshot (preserving binary/env/unit
|
||||||
|
# metadata), restores the prior enable state (or disables a freshly-installed
|
||||||
|
# unit on a fresh install), and brings the prior service back up. The trap
|
||||||
|
# also fires on HUP/INT/TERM so an interrupted run rolls back instead of
|
||||||
|
# leaving a half-activated state.
|
||||||
|
rolled_back=0
|
||||||
|
rollback() {
|
||||||
|
[ "$rolled_back" -eq 1 ] && return 0
|
||||||
|
rolled_back=1
|
||||||
|
echo "rsmon-worker activation failed; restoring the prior install" >&2
|
||||||
|
stop_svc || true
|
||||||
|
rm -f "$binary.new" "$env_file.new" "$unit_file.new" "$pid_file" || true
|
||||||
|
if [ "$had_binary" -eq 1 ]; then
|
||||||
|
cp -p "$backup_dir/binary" "$binary" || true
|
||||||
|
elif [ -e "$binary" ]; then
|
||||||
|
rm -f "$binary" || true
|
||||||
|
fi
|
||||||
|
if [ "$had_env" -eq 1 ]; then
|
||||||
|
cp -p "$backup_dir/worker.env" "$env_file" || true
|
||||||
|
chmod 0600 "$env_file" || true
|
||||||
|
elif [ -e "$env_file" ]; then
|
||||||
|
rm -f "$env_file" || true
|
||||||
|
fi
|
||||||
|
if [ -n "$unit_file" ]; then
|
||||||
|
if [ "$had_unit" -eq 1 ]; then
|
||||||
|
cp -p "$backup_dir/unit" "$unit_file" || true
|
||||||
|
chmod "$unit_mode" "$unit_file" || true
|
||||||
|
set_unit_enabled "$unit_enabled" || true
|
||||||
|
else
|
||||||
|
set_unit_enabled 0 || true
|
||||||
|
rm -f "$unit_file" || true
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
rm -f "$env_tmp" "$unit_tmp" || true
|
||||||
|
rm -rf "$backup_dir" || true
|
||||||
|
if [ "$had_binary" -eq 1 ] && [ "$had_env" -eq 1 ] && [ "$no_start" -ne 1 ]; then
|
||||||
|
restart_svc || true
|
||||||
|
fi
|
||||||
|
release_lock
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
trap rollback EXIT HUP INT TERM
|
||||||
|
|
||||||
|
# --- atomic binary install ------------------------------------------------
|
||||||
|
mkdir -p "$(dirname "$binary")"
|
||||||
|
install -m 0755 "$stage" "$binary.new"
|
||||||
|
mv -f "$binary.new" "$binary"
|
||||||
|
|
||||||
|
# --- atomic env install (secrets, mode 0600) ------------------------------
|
||||||
|
mkdir -p "$config_dir"
|
||||||
|
chmod 0750 "$config_dir"
|
||||||
|
install -m 0600 "$env_tmp" "$env_file.new"
|
||||||
|
mv -f "$env_file.new" "$env_file"
|
||||||
|
rm -f "$env_tmp"
|
||||||
|
|
||||||
|
# --- data directory -------------------------------------------------------
|
||||||
|
mkdir -p "$data_dir" "$data_dir/webapp"
|
||||||
|
chmod 0755 "$data_dir" "$data_dir/webapp"
|
||||||
|
|
||||||
|
# --- service definition ---------------------------------------------------
|
||||||
|
if [ -n "$unit_file" ] && [ -f "$unit_tmp" ]; then
|
||||||
|
mkdir -p "$(dirname "$unit_file")"
|
||||||
|
install -m "$unit_mode" "$unit_tmp" "$unit_file.new"
|
||||||
|
mv -f "$unit_file.new" "$unit_file"
|
||||||
|
rm -f "$unit_tmp"
|
||||||
|
set_unit_enabled 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
# --- start and verify -----------------------------------------------------
|
||||||
|
# A failed start or a worker that does not answer /healthz triggers the
|
||||||
|
# rollback trap.
|
||||||
|
if [ "$no_start" -ne 1 ]; then
|
||||||
|
if ! restart_svc; then
|
||||||
|
echo "rsmon-worker activation failed: process or /healthz verification failed" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
else
|
||||||
|
stop_svc || true
|
||||||
|
fi
|
||||||
|
|
||||||
|
# --- success --------------------------------------------------------------
|
||||||
|
rm -rf "$backup_dir"
|
||||||
|
rm -f "$env_tmp" "$unit_tmp"
|
||||||
|
release_lock
|
||||||
|
trap - EXIT HUP INT TERM
|
||||||
|
if [ "$no_start" -eq 1 ]; then
|
||||||
|
echo "rsmon-worker activated: binary=$binary supervisor=$supervisor unit=${unit_file:-none} started=no"
|
||||||
|
else
|
||||||
|
echo "rsmon-worker activated: binary=$binary supervisor=$supervisor unit=${unit_file:-none} started=yes"
|
||||||
|
fi
|
||||||
|
exit 0
|
||||||
|
`
|
||||||
|
|
||||||
|
// activateParams are the resolved, controller-validated values fed into
|
||||||
|
// the remote activation script.
|
||||||
|
type activateParams struct {
|
||||||
|
Stage string
|
||||||
|
Binary string
|
||||||
|
ConfigDir string
|
||||||
|
EnvFile string
|
||||||
|
EnvTmp string
|
||||||
|
DataDir string
|
||||||
|
UnitTmp string
|
||||||
|
UnitFile string
|
||||||
|
UnitMode string
|
||||||
|
UnitName string // systemd unit name (rsmon-worker.service)
|
||||||
|
RCName string // rc-service name (rsmon-worker)
|
||||||
|
Supervisor string // systemd | openrc | none
|
||||||
|
NoStart string // 0 or 1
|
||||||
|
}
|
||||||
|
|
||||||
|
// activateWorker performs the work-package-4 activation for an already
|
||||||
|
// staged source build over the live SSH executor. It uses the env bytes
|
||||||
|
// rendered once during option normalization, uploads the env and service
|
||||||
|
// definition into a server-created 0700 temp dir (no /tmp symlink/TOCTOU
|
||||||
|
// attack surface), runs the atomic activation script, and records where
|
||||||
|
// and how the worker was activated. Every remote temp file it creates is
|
||||||
|
// removed on success and failure.
|
||||||
|
func (e *sourceExecutor) activateWorker(opts SourceInstallOptions, res *SourceInstallResult) error {
|
||||||
|
a := opts.Activation
|
||||||
|
name := strings.TrimSpace(a.Name)
|
||||||
|
p := resolvePaths(name)
|
||||||
|
|
||||||
|
envData := opts.activationEnv
|
||||||
|
if len(envData) == 0 {
|
||||||
|
return errors.New("activation environment was not rendered (internal error)")
|
||||||
|
}
|
||||||
|
|
||||||
|
unitContent, unitFile, unitMode := unitContent(res.Detection.InitSystem, p)
|
||||||
|
svcName := strings.TrimSuffix(p.unitName, ".service")
|
||||||
|
|
||||||
|
supOut, err := e.runPlain("sh -c " + shellQuote(runningInitScript))
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("detect running init system: %w", err)
|
||||||
|
}
|
||||||
|
supervisor := normalizeSupervisor(string(supOut))
|
||||||
|
|
||||||
|
// Create a server-side 0700 temp dir owned by the SSH user, then
|
||||||
|
// upload the env/unit into it. mktemp -d produces an unpredictable,
|
||||||
|
// private path, so a hostile local user cannot pre-create a symlink at
|
||||||
|
// a predictable /tmp name (the classic upload TOCTOU). The token
|
||||||
|
// never appears in argv or logs: it travels only as base64 over the
|
||||||
|
// session stdin and later only inside the mode-0600 env file.
|
||||||
|
out, err := e.runPlain("d=$(mktemp -d /tmp/rsmon-worker-act.XXXXXX) && chmod 0700 \"$d\" && echo \"$d\"")
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("create secure upload directory: %w", err)
|
||||||
|
}
|
||||||
|
secureDir := strings.TrimSpace(string(out))
|
||||||
|
if secureDir == "" || !strings.HasPrefix(secureDir, "/tmp/") {
|
||||||
|
return errors.New("remote returned an invalid secure upload directory")
|
||||||
|
}
|
||||||
|
envTmp := secureDir + "/worker.env"
|
||||||
|
unitTmp := secureDir + "/unit"
|
||||||
|
if err := uploadBytes(e.client, envData, envTmp, 0o600); err != nil {
|
||||||
|
return fmt.Errorf("upload worker environment: %w", err)
|
||||||
|
}
|
||||||
|
if unitContent != "" {
|
||||||
|
if err := uploadBytes(e.client, []byte(unitContent), unitTmp, 0o644); err != nil {
|
||||||
|
return fmt.Errorf("upload service definition: %w", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
defer func() {
|
||||||
|
_ = runRemote(e.client, "rm -rf -- "+shellQuote(secureDir), nil)
|
||||||
|
}()
|
||||||
|
|
||||||
|
noStart := "0"
|
||||||
|
if a.NoStart {
|
||||||
|
noStart = "1"
|
||||||
|
}
|
||||||
|
script := activateScript(activateParams{
|
||||||
|
Stage: res.StageBinary,
|
||||||
|
Binary: p.binary,
|
||||||
|
ConfigDir: p.configDir,
|
||||||
|
EnvFile: p.envFile,
|
||||||
|
EnvTmp: envTmp,
|
||||||
|
DataDir: p.dataDir,
|
||||||
|
UnitTmp: unitTmp,
|
||||||
|
UnitFile: unitFile,
|
||||||
|
UnitMode: unitMode,
|
||||||
|
UnitName: p.unitName,
|
||||||
|
RCName: svcName,
|
||||||
|
Supervisor: supervisor,
|
||||||
|
NoStart: noStart,
|
||||||
|
})
|
||||||
|
if _, err := e.runPrivileged("sh -c " + shellQuote(script)); err != nil {
|
||||||
|
return fmt.Errorf("activate worker service: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
res.Activation = &ActivationResult{
|
||||||
|
Binary: p.binary,
|
||||||
|
ConfigDir: p.configDir,
|
||||||
|
EnvFile: p.envFile,
|
||||||
|
DataDir: p.dataDir,
|
||||||
|
UnitFile: unitFile,
|
||||||
|
UnitName: p.unitName,
|
||||||
|
Supervisor: supervisor,
|
||||||
|
Started: !a.NoStart,
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// validateRenderedEnv runs the strict environment-file parser over the
|
||||||
|
// rendered env bytes in memory so the exact bytes written remotely pass
|
||||||
|
// the same gate as a user-supplied file, without re-reading a file.
|
||||||
|
func validateRenderedEnv(data []byte) error {
|
||||||
|
_, err := parseEnvironmentContent(data)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// openrcInitFor renders the native OpenRC init script for an instance. It
|
||||||
|
// mirrors the hardening of the systemd unit (data dir, env file, webapp
|
||||||
|
// data dir) using OpenRC conventions; it is written for real hosts where
|
||||||
|
// openrc is the running init, while containers that cannot run openrc
|
||||||
|
// fall back to the embedded supervisor.
|
||||||
|
func openrcInitFor(p paths) string {
|
||||||
|
description := "RSMon distributed monitoring worker"
|
||||||
|
svcName := strings.TrimSuffix(p.unitName, ".service")
|
||||||
|
if p.name != "" {
|
||||||
|
description += " (" + p.name + ")"
|
||||||
|
}
|
||||||
|
return fmt.Sprintf(`#!/sbin/openrc-run
|
||||||
|
# Managed by the rsmon-worker source installer; do not edit by hand.
|
||||||
|
|
||||||
|
name=%s
|
||||||
|
description=%s
|
||||||
|
command=%s
|
||||||
|
command_background=true
|
||||||
|
pidfile=%s/worker.pid
|
||||||
|
output_log=%s/worker.log
|
||||||
|
error_log=%s/worker.log
|
||||||
|
|
||||||
|
depend() {
|
||||||
|
need net
|
||||||
|
}
|
||||||
|
|
||||||
|
start_pre() {
|
||||||
|
checkpath --directory --mode 0755 --owner root:root %s %s/webapp
|
||||||
|
if [ -f %s ]; then
|
||||||
|
# OpenRC runs the command with the init script's environment, so
|
||||||
|
# the env-file variables must be exported (set -a) or the worker
|
||||||
|
# would start without them.
|
||||||
|
set -a
|
||||||
|
. %s
|
||||||
|
set +a
|
||||||
|
fi
|
||||||
|
export HOME=%s
|
||||||
|
export RSMON_WEBAPP_DATA_DIR=%s/webapp
|
||||||
|
}
|
||||||
|
`, svcName, description, p.binary, p.dataDir, p.dataDir, p.dataDir,
|
||||||
|
p.dataDir, p.dataDir, p.envFile, p.envFile, p.dataDir, p.dataDir)
|
||||||
|
}
|
||||||
438
internal/installer/sourceactivate_shell_test.go
Обычный файл
438
internal/installer/sourceactivate_shell_test.go
Обычный файл
@@ -0,0 +1,438 @@
|
|||||||
|
package installer
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os"
|
||||||
|
"os/exec"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
// This file exercises the rendered activation script against stub
|
||||||
|
// systemd/OpenRC implementations ("targeted shell fixture tests"). Real
|
||||||
|
// systemd and OpenRC cannot run inside the Docker/OpenSSH harness (the
|
||||||
|
// container PID 1 is sshd), so these tests execute the actual remote
|
||||||
|
// script on the host with fake `systemctl`, `rc-service`, and `rc-update`
|
||||||
|
// binaries that simulate an init-managed host. This covers the unit
|
||||||
|
// install/enable, the supervisor-driven restart, the `is-active`-based
|
||||||
|
// health loop, the enable-state rollback, the activation lock, and the
|
||||||
|
// interrupted-run recovery paths that the E2E fixtures cannot reach.
|
||||||
|
//
|
||||||
|
// Residual limitation: the stub init tools verify command flow and state
|
||||||
|
// transitions, not the real systemd/OpenRC unit semantics; a real
|
||||||
|
// init-system smoke test remains out of scope for the container harness.
|
||||||
|
|
||||||
|
// systemctlStub records every invocation and simulates unit state in a
|
||||||
|
// fake state directory: `is-active`/`is-enabled` consult marker files,
|
||||||
|
// `restart` marks the unit active (or fails when the fail-restart marker
|
||||||
|
// is present), and `enable`/`disable` toggle the enabled marker.
|
||||||
|
const systemctlStub = `#!/bin/sh
|
||||||
|
echo "$*" >> "$SYSCTL_LOG"
|
||||||
|
action="$1"
|
||||||
|
shift
|
||||||
|
unit=""
|
||||||
|
for a in "$@"; do
|
||||||
|
case "$a" in --*) continue ;; esac
|
||||||
|
unit="$a"
|
||||||
|
break
|
||||||
|
done
|
||||||
|
case "$action" in
|
||||||
|
is-active) [ -f "$FAKE_SYSTEMD_DIR/$unit.active" ] || exit 3 ;;
|
||||||
|
is-enabled) [ -f "$FAKE_SYSTEMD_DIR/$unit.enabled" ] || exit 1 ;;
|
||||||
|
enable) touch "$FAKE_SYSTEMD_DIR/$unit.enabled"; exit 0 ;;
|
||||||
|
disable) rm -f "$FAKE_SYSTEMD_DIR/$unit.enabled"; exit 0 ;;
|
||||||
|
daemon-reload) exit 0 ;;
|
||||||
|
restart) [ -f "$FAKE_SYSTEMD_DIR/fail-restart" ] && exit 1
|
||||||
|
touch "$FAKE_SYSTEMD_DIR/$unit.active"; exit 0 ;;
|
||||||
|
stop) rm -f "$FAKE_SYSTEMD_DIR/$unit.active"; exit 0 ;;
|
||||||
|
*) exit 0 ;;
|
||||||
|
esac
|
||||||
|
`
|
||||||
|
|
||||||
|
// rcServiceStub simulates OpenRC service state; rcUpdateStub simulates
|
||||||
|
// runlevel enablement.
|
||||||
|
const rcServiceStub = `#!/bin/sh
|
||||||
|
echo "$*" >> "$RC_LOG"
|
||||||
|
svc="$1"
|
||||||
|
action="$2"
|
||||||
|
case "$action" in
|
||||||
|
restart) [ -f "$FAKE_RC_DIR/fail-restart" ] && exit 1
|
||||||
|
touch "$FAKE_RC_DIR/$svc.active"; exit 0 ;;
|
||||||
|
status) [ -f "$FAKE_RC_DIR/$svc.active" ] || exit 1 ;;
|
||||||
|
stop) rm -f "$FAKE_RC_DIR/$svc.active"; exit 0 ;;
|
||||||
|
*) exit 0 ;;
|
||||||
|
esac
|
||||||
|
`
|
||||||
|
|
||||||
|
const rcUpdateStub = `#!/bin/sh
|
||||||
|
echo "$*" >> "$RCU_LOG"
|
||||||
|
case "$1" in
|
||||||
|
add) touch "$FAKE_RC_DIR/$2.enabled"; exit 0 ;;
|
||||||
|
del) rm -f "$FAKE_RC_DIR/$2.enabled"; exit 0 ;;
|
||||||
|
*) exit 0 ;;
|
||||||
|
esac
|
||||||
|
`
|
||||||
|
|
||||||
|
// workerStub stands in for the built worker binary: it satisfies the
|
||||||
|
// `--version` validation and the `/healthz` liveness probe.
|
||||||
|
const workerStub = `#!/bin/sh
|
||||||
|
if [ "$1" = "--version" ]; then
|
||||||
|
echo "rsmon-worker version=dev commit=deadbeefdead buildDate=2026-01-01T00:00:00Z"
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
if [ "$1" = "liveness" ]; then
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
exit 1
|
||||||
|
`
|
||||||
|
|
||||||
|
// initShellFixture builds a temp "host" with stub init tools and a stub
|
||||||
|
// worker, plus the uploaded env/unit the activation script consumes.
|
||||||
|
type initShellFixture struct {
|
||||||
|
root string
|
||||||
|
binDir string
|
||||||
|
sysdDir string
|
||||||
|
rcDir string
|
||||||
|
|
||||||
|
sysctlLog string
|
||||||
|
rcLog string
|
||||||
|
rcuLog string
|
||||||
|
|
||||||
|
params activateParams
|
||||||
|
}
|
||||||
|
|
||||||
|
func writeExec(t *testing.T, path, content string) {
|
||||||
|
t.Helper()
|
||||||
|
if err := os.WriteFile(path, []byte(content), 0o700); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func newInitShellFixture(t *testing.T, supervisor string) *initShellFixture {
|
||||||
|
t.Helper()
|
||||||
|
root := t.TempDir()
|
||||||
|
fx := &initShellFixture{
|
||||||
|
root: root,
|
||||||
|
binDir: filepath.Join(root, "bin"),
|
||||||
|
sysdDir: filepath.Join(root, "sysd"),
|
||||||
|
rcDir: filepath.Join(root, "rc"),
|
||||||
|
sysctlLog: filepath.Join(root, "sysctl.log"),
|
||||||
|
rcLog: filepath.Join(root, "rc.log"),
|
||||||
|
rcuLog: filepath.Join(root, "rcu.log"),
|
||||||
|
}
|
||||||
|
for _, d := range []string{
|
||||||
|
fx.binDir, fx.sysdDir, fx.rcDir, filepath.Join(root, "upload"),
|
||||||
|
filepath.Join(root, "usr/local/bin"), filepath.Join(root, "etc/rsmon-worker"),
|
||||||
|
} {
|
||||||
|
if err := os.MkdirAll(d, 0o755); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
writeExec(t, filepath.Join(fx.binDir, "systemctl"), systemctlStub)
|
||||||
|
writeExec(t, filepath.Join(fx.binDir, "rc-service"), rcServiceStub)
|
||||||
|
writeExec(t, filepath.Join(fx.binDir, "rc-update"), rcUpdateStub)
|
||||||
|
|
||||||
|
stage := filepath.Join(root, "usr/local/bin/rsmon-worker.stage")
|
||||||
|
writeExec(t, stage, workerStub)
|
||||||
|
|
||||||
|
envTmp := filepath.Join(root, "upload", "worker.env")
|
||||||
|
unitTmp := filepath.Join(root, "upload", "unit")
|
||||||
|
if err := os.WriteFile(envTmp, []byte(
|
||||||
|
"RSMON_URL=https://rsmon.ru\nRSMON_TOKEN=shell-test-token\nWORKER_HOST=127.0.0.1\nWORKER_PORT=27401\n",
|
||||||
|
), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
unitFile := filepath.Join(root, "etc/systemd/system/rsmon-worker.service")
|
||||||
|
unitMode := "0644"
|
||||||
|
if supervisor == "openrc" {
|
||||||
|
unitFile = filepath.Join(root, "etc/init.d/rsmon-worker")
|
||||||
|
unitMode = "0755"
|
||||||
|
if err := os.MkdirAll(filepath.Dir(unitFile), 0o755); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(unitTmp, []byte("# fake openrc unit\n"), 0o755); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
if err := os.MkdirAll(filepath.Dir(unitFile), 0o755); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(unitTmp, []byte("[Unit]\n# fake systemd unit\n"), 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
dataDir := filepath.Join(root, "var/lib/rsmon-worker")
|
||||||
|
fx.params = activateParams{
|
||||||
|
Stage: stage,
|
||||||
|
Binary: filepath.Join(root, "usr/local/bin/rsmon-worker"),
|
||||||
|
ConfigDir: filepath.Join(root, "etc/rsmon-worker"),
|
||||||
|
EnvFile: filepath.Join(root, "etc/rsmon-worker/worker.env"),
|
||||||
|
EnvTmp: envTmp,
|
||||||
|
DataDir: dataDir,
|
||||||
|
UnitTmp: unitTmp,
|
||||||
|
UnitFile: unitFile,
|
||||||
|
UnitMode: unitMode,
|
||||||
|
UnitName: "rsmon-worker.service",
|
||||||
|
RCName: "rsmon-worker",
|
||||||
|
Supervisor: supervisor,
|
||||||
|
NoStart: "0",
|
||||||
|
}
|
||||||
|
return fx
|
||||||
|
}
|
||||||
|
|
||||||
|
// run executes the rendered activation script with the stub init tools
|
||||||
|
// first in PATH.
|
||||||
|
func (fx *initShellFixture) run(t *testing.T) (string, error) {
|
||||||
|
t.Helper()
|
||||||
|
script := activateScript(fx.params)
|
||||||
|
cmd := exec.Command("sh", "-c", script)
|
||||||
|
cmd.Env = append(
|
||||||
|
os.Environ(),
|
||||||
|
"PATH="+fx.binDir+":"+os.Getenv("PATH"),
|
||||||
|
"FAKE_SYSTEMD_DIR="+fx.sysdDir,
|
||||||
|
"FAKE_RC_DIR="+fx.rcDir,
|
||||||
|
"SYSCTL_LOG="+fx.sysctlLog,
|
||||||
|
"RC_LOG="+fx.rcLog,
|
||||||
|
"RCU_LOG="+fx.rcuLog,
|
||||||
|
)
|
||||||
|
out, err := cmd.CombinedOutput()
|
||||||
|
return string(out), err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (fx *initShellFixture) log(t *testing.T, path string) string {
|
||||||
|
t.Helper()
|
||||||
|
b, err := os.ReadFile(path)
|
||||||
|
if err != nil {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return string(b)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (fx *initShellFixture) assertNoLeftovers(t *testing.T) {
|
||||||
|
t.Helper()
|
||||||
|
for _, name := range []string{".rsmon-backup", ".rsmon-activate.lock"} {
|
||||||
|
if _, err := os.Stat(filepath.Join(fx.params.DataDir, name)); !os.IsNotExist(err) {
|
||||||
|
t.Fatalf("%s left behind after activation", name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestActivateShellSystemd runs the real activation script against a stub
|
||||||
|
// systemd host and verifies the init-managed flow: the unit is installed
|
||||||
|
// and enabled, restart goes through systemctl, and the health loop uses
|
||||||
|
// `systemctl is-active` (no pid file is ever created).
|
||||||
|
func TestActivateShellSystemd(t *testing.T) {
|
||||||
|
fx := newInitShellFixture(t, "systemd")
|
||||||
|
out, err := fx.run(t)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("systemd activation failed: %v\n%s", err, out)
|
||||||
|
}
|
||||||
|
if !strings.Contains(out, "rsmon-worker activated") {
|
||||||
|
t.Fatalf("no activation success line:\n%s", out)
|
||||||
|
}
|
||||||
|
unit, err := os.ReadFile(fx.params.UnitFile)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("unit not installed: %v", err)
|
||||||
|
}
|
||||||
|
if !strings.Contains(string(unit), "[Unit]") {
|
||||||
|
t.Fatalf("installed unit has wrong content: %q", unit)
|
||||||
|
}
|
||||||
|
st, err := os.Stat(fx.params.EnvFile)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("env not installed: %v", err)
|
||||||
|
}
|
||||||
|
if st.Mode().Perm() != 0o600 {
|
||||||
|
t.Fatalf("env mode = %v, want 0600", st.Mode().Perm())
|
||||||
|
}
|
||||||
|
log := fx.log(t, fx.sysctlLog)
|
||||||
|
for _, want := range []string{
|
||||||
|
"enable rsmon-worker.service",
|
||||||
|
"restart rsmon-worker.service",
|
||||||
|
"is-active --quiet rsmon-worker.service",
|
||||||
|
} {
|
||||||
|
if !strings.Contains(log, want) {
|
||||||
|
t.Fatalf("systemctl log missing %q:\n%s", want, log)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// The health loop used systemctl is-active, never the embedded
|
||||||
|
// supervisor's pid file.
|
||||||
|
if _, err := os.Stat(filepath.Join(fx.params.DataDir, "worker.pid")); !os.IsNotExist(err) {
|
||||||
|
t.Fatalf("systemd activation created a pid file; the health loop must use systemctl is-active")
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(filepath.Join(fx.sysdDir, "rsmon-worker.service.enabled")); err != nil {
|
||||||
|
t.Fatalf("unit not enabled: %v", err)
|
||||||
|
}
|
||||||
|
fx.assertNoLeftovers(t)
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestActivateShellOpenRC runs the real activation script against a stub
|
||||||
|
// OpenRC host: the init script is installed executable, the runlevel
|
||||||
|
// enablement and rc-service restart/status are used, and the health loop
|
||||||
|
// goes through rc-service (no pid file).
|
||||||
|
func TestActivateShellOpenRC(t *testing.T) {
|
||||||
|
fx := newInitShellFixture(t, "openrc")
|
||||||
|
out, err := fx.run(t)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("openrc activation failed: %v\n%s", err, out)
|
||||||
|
}
|
||||||
|
if !strings.Contains(out, "rsmon-worker activated") {
|
||||||
|
t.Fatalf("no activation success line:\n%s", out)
|
||||||
|
}
|
||||||
|
st, err := os.Stat(fx.params.UnitFile)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("openrc init not installed: %v", err)
|
||||||
|
}
|
||||||
|
if st.Mode().Perm() != 0o755 {
|
||||||
|
t.Fatalf("openrc init mode = %v, want 0755", st.Mode().Perm())
|
||||||
|
}
|
||||||
|
rcLog := fx.log(t, fx.rcLog)
|
||||||
|
for _, want := range []string{"rsmon-worker restart", "rsmon-worker status"} {
|
||||||
|
if !strings.Contains(rcLog, want) {
|
||||||
|
t.Fatalf("rc-service log missing %q:\n%s", want, rcLog)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
rcuLog := fx.log(t, fx.rcuLog)
|
||||||
|
if !strings.Contains(rcuLog, "add rsmon-worker default") {
|
||||||
|
t.Fatalf("rc-update add not issued:\n%s", rcuLog)
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(filepath.Join(fx.rcDir, "rsmon-worker.enabled")); err != nil {
|
||||||
|
t.Fatalf("service not enabled in the runlevel: %v", err)
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(filepath.Join(fx.params.DataDir, "worker.pid")); !os.IsNotExist(err) {
|
||||||
|
t.Fatalf("openrc activation created a pid file; the health loop must use rc-service status")
|
||||||
|
}
|
||||||
|
fx.assertNoLeftovers(t)
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestActivateShellSystemdRollbackFreshDisablesUnit proves a failed
|
||||||
|
// activation on a fresh host disables the newly-enabled unit and removes
|
||||||
|
// the unit file (no dangling enablement).
|
||||||
|
func TestActivateShellSystemdRollbackFreshDisablesUnit(t *testing.T) {
|
||||||
|
fx := newInitShellFixture(t, "systemd")
|
||||||
|
if err := os.WriteFile(filepath.Join(fx.sysdDir, "fail-restart"), []byte(""), 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
out, err := fx.run(t)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("activation succeeded despite restart failure")
|
||||||
|
}
|
||||||
|
if !strings.Contains(out, "start failure") {
|
||||||
|
t.Fatalf("no start-failure classification:\n%s", out)
|
||||||
|
}
|
||||||
|
log := fx.log(t, fx.sysctlLog)
|
||||||
|
if !strings.Contains(log, "disable rsmon-worker.service") {
|
||||||
|
t.Fatalf("fresh rollback did not disable the unit:\n%s", log)
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(fx.params.UnitFile); !os.IsNotExist(err) {
|
||||||
|
t.Fatal("unit file left behind after fresh rollback")
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(filepath.Join(fx.sysdDir, "rsmon-worker.service.enabled")); !os.IsNotExist(err) {
|
||||||
|
t.Fatal("unit still enabled after fresh rollback")
|
||||||
|
}
|
||||||
|
fx.assertNoLeftovers(t)
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestActivateShellSystemdRollbackRestoresEnableState proves a failed
|
||||||
|
// activation on a host with a prior enabled unit restores the unit
|
||||||
|
// byte-for-byte and re-applies the prior enabled state.
|
||||||
|
func TestActivateShellSystemdRollbackRestoresEnableState(t *testing.T) {
|
||||||
|
fx := newInitShellFixture(t, "systemd")
|
||||||
|
if err := os.WriteFile(fx.params.UnitFile, []byte("PRIOR-UNIT-CONTENT"), 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(filepath.Join(fx.sysdDir, "rsmon-worker.service.enabled"), []byte(""), 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(filepath.Join(fx.sysdDir, "fail-restart"), []byte(""), 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
out, err := fx.run(t)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("activation succeeded despite restart failure")
|
||||||
|
}
|
||||||
|
if !strings.Contains(out, "start failure") {
|
||||||
|
t.Fatalf("no start-failure classification:\n%s", out)
|
||||||
|
}
|
||||||
|
unit, err := os.ReadFile(fx.params.UnitFile)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if string(unit) != "PRIOR-UNIT-CONTENT" {
|
||||||
|
t.Fatalf("prior unit not restored: %q", unit)
|
||||||
|
}
|
||||||
|
log := fx.log(t, fx.sysctlLog)
|
||||||
|
if !strings.Contains(log, "enable rsmon-worker.service") {
|
||||||
|
t.Fatalf("prior enabled state not re-applied:\n%s", log)
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(filepath.Join(fx.sysdDir, "rsmon-worker.service.enabled")); err != nil {
|
||||||
|
t.Fatalf("unit not re-enabled after rollback: %v", err)
|
||||||
|
}
|
||||||
|
fx.assertNoLeftovers(t)
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestActivateShellStaleLockBroken proves a leftover lock from a killed
|
||||||
|
// run (dead pid) is broken automatically and the activation proceeds.
|
||||||
|
func TestActivateShellStaleLockBroken(t *testing.T) {
|
||||||
|
fx := newInitShellFixture(t, "systemd")
|
||||||
|
lockDir := filepath.Join(fx.params.DataDir, ".rsmon-activate.lock")
|
||||||
|
if err := os.MkdirAll(lockDir, 0o700); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(filepath.Join(lockDir, "pid"), []byte("999999\n"), 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
out, err := fx.run(t)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("activation failed with a stale lock: %v\n%s", err, out)
|
||||||
|
}
|
||||||
|
if !strings.Contains(out, "rsmon-worker activated") {
|
||||||
|
t.Fatalf("no activation success line:\n%s", out)
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(lockDir); !os.IsNotExist(err) {
|
||||||
|
t.Fatal("lock not released after activation")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestActivateShellRecoveryFromInterruptedBackup proves the interrupted-run
|
||||||
|
// recovery: a leftover backup marker from a killed activation is restored
|
||||||
|
// before the fresh run, so a later validation failure still leaves the
|
||||||
|
// recovered prior install in place.
|
||||||
|
func TestActivateShellRecoveryFromInterruptedBackup(t *testing.T) {
|
||||||
|
fx := newInitShellFixture(t, "none")
|
||||||
|
backupDir := filepath.Join(fx.params.DataDir, ".rsmon-backup")
|
||||||
|
if err := os.MkdirAll(backupDir, 0o700); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(filepath.Join(backupDir, "binary"), []byte("RECOVERED-BINARY"), 0o755); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(filepath.Join(backupDir, "worker.env"), []byte("RSMON_URL=https://rsmon.ru\nRSMON_TOKEN=test\n"), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(filepath.Join(backupDir, ".marker"), []byte("0"), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
// Corrupt the stage so the fresh activation fails at validation (before
|
||||||
|
// the fresh snapshot), proving the recovered files were already in place.
|
||||||
|
if err := os.WriteFile(fx.params.Stage, []byte("not a script"), 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
out, err := fx.run(t)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("activation succeeded with a corrupt stage")
|
||||||
|
}
|
||||||
|
if !strings.Contains(out, "recovering interrupted activation") {
|
||||||
|
t.Fatalf("recovery did not run:\n%s", out)
|
||||||
|
}
|
||||||
|
b, err := os.ReadFile(fx.params.Binary)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("recovered binary not restored: %v", err)
|
||||||
|
}
|
||||||
|
if string(b) != "RECOVERED-BINARY" {
|
||||||
|
t.Fatalf("recovered binary content = %q", b)
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(backupDir); !os.IsNotExist(err) {
|
||||||
|
t.Fatal("backup not consumed by recovery")
|
||||||
|
}
|
||||||
|
}
|
||||||
409
internal/installer/sourceactivate_test.go
Обычный файл
409
internal/installer/sourceactivate_test.go
Обычный файл
@@ -0,0 +1,409 @@
|
|||||||
|
package installer
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"rocketgit.ru/rsmon/worker/internal/sshinstall"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestActivateScriptMarkers(t *testing.T) {
|
||||||
|
script := activateScript(activateParams{
|
||||||
|
Stage: "/opt/rsmon-worker-src/rsmon-worker",
|
||||||
|
Binary: "/usr/local/bin/rsmon-worker",
|
||||||
|
ConfigDir: "/etc/rsmon-worker",
|
||||||
|
EnvFile: "/etc/rsmon-worker/worker.env",
|
||||||
|
EnvTmp: "/tmp/rsmon-worker-abc.env",
|
||||||
|
DataDir: "/var/lib/rsmon-worker",
|
||||||
|
UnitTmp: "/tmp/rsmon-worker-abc.service",
|
||||||
|
UnitFile: "/etc/systemd/system/rsmon-worker.service",
|
||||||
|
UnitMode: "0644",
|
||||||
|
UnitName: "rsmon-worker.service",
|
||||||
|
RCName: "rsmon-worker",
|
||||||
|
Supervisor: "none",
|
||||||
|
NoStart: "0",
|
||||||
|
})
|
||||||
|
for _, want := range []string{
|
||||||
|
"set -eu",
|
||||||
|
// staged binary validated before any state is touched
|
||||||
|
"\"$stage\" --version >/dev/null",
|
||||||
|
// snapshot + rollback
|
||||||
|
"backup_dir=\"$data_dir/.rsmon-backup\"",
|
||||||
|
"rm -rf \"$backup_dir\"",
|
||||||
|
"cp -p \"$binary\" \"$backup_dir/binary\"",
|
||||||
|
"cp -p \"$env_file\" \"$backup_dir/worker.env\"",
|
||||||
|
"printf '%s\\n' \"$unit_enabled\" > \"$backup_dir/.marker\"",
|
||||||
|
"trap rollback EXIT HUP INT TERM",
|
||||||
|
"restoring the prior install",
|
||||||
|
// atomic installs with the right perms
|
||||||
|
"install -m 0755 \"$stage\" \"$binary.new\"",
|
||||||
|
"mv -f \"$binary.new\" \"$binary\"",
|
||||||
|
"install -m 0600 \"$env_tmp\" \"$env_file.new\"",
|
||||||
|
"mv -f \"$env_file.new\" \"$env_file\"",
|
||||||
|
"chmod 0750 \"$config_dir\"",
|
||||||
|
"mkdir -p \"$data_dir\" \"$data_dir/webapp\"",
|
||||||
|
"install -m \"$unit_mode\" \"$unit_tmp\" \"$unit_file.new\"",
|
||||||
|
"mv -f \"$unit_file.new\" \"$unit_file\"",
|
||||||
|
// supervisor + pid + health verification
|
||||||
|
"pid_file=\"$data_dir/worker.pid\"",
|
||||||
|
"env -i PATH=\"/usr/bin:/bin:/sbin:/usr/sbin\"",
|
||||||
|
"RSMON_WORKER_ENV_FILE=\"$env_file\"",
|
||||||
|
". \"$RSMON_WORKER_ENV_FILE\"",
|
||||||
|
"exec \"$@\"",
|
||||||
|
"nohup \"$1\" >>\"$2\" 2>&1 & echo $! > \"$3\"",
|
||||||
|
"run_env \"$binary\" liveness >/dev/null 2>&1",
|
||||||
|
"while [ \"$i\" -lt 30 ]; do",
|
||||||
|
"rsmon-worker activated:",
|
||||||
|
// activation lock + interrupted-run recovery
|
||||||
|
"lock_dir=\"$data_dir/.rsmon-activate.lock\"",
|
||||||
|
"acquire_lock",
|
||||||
|
"another rsmon-worker activation is in progress",
|
||||||
|
"recovering interrupted activation from $backup_dir",
|
||||||
|
".marker",
|
||||||
|
// pid belongs to the expected binary before kill
|
||||||
|
"readlink \"/proc/$1/exe\"",
|
||||||
|
"\"$binary (deleted)\"",
|
||||||
|
"process_is_worker \"$pid\"",
|
||||||
|
// rollback preserves metadata and enable state
|
||||||
|
"cp -p \"$backup_dir/binary\" \"$binary\"",
|
||||||
|
"cp -p \"$backup_dir/worker.env\" \"$env_file\"",
|
||||||
|
"set_unit_enabled \"$unit_enabled\"",
|
||||||
|
"systemctl is-enabled",
|
||||||
|
// temp/backup cleanup on success
|
||||||
|
"rm -rf \"$backup_dir\"",
|
||||||
|
"release_lock",
|
||||||
|
"trap - EXIT HUP INT TERM",
|
||||||
|
} {
|
||||||
|
if !strings.Contains(script, want) {
|
||||||
|
t.Fatalf("activate script missing %q:\n%s", want, script)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// The rollback must be armed only after the snapshot so a corrupt
|
||||||
|
// staging binary (validated first) never triggers a destructive
|
||||||
|
// rollback of the prior install.
|
||||||
|
if !strings.Contains(script, "\"$stage\" --version") || !strings.Contains(script, "trap rollback EXIT HUP INT TERM") {
|
||||||
|
t.Fatalf("activate script must validate the staged binary before arming rollback:\n%s", script)
|
||||||
|
}
|
||||||
|
stageCheck := strings.Index(script, "\"$stage\" --version")
|
||||||
|
trapIdx := strings.Index(script, "trap rollback EXIT HUP INT TERM")
|
||||||
|
if stageCheck < 0 || trapIdx < stageCheck {
|
||||||
|
t.Fatalf("staged-binary validation must precede the rollback trap:\n%s", script)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestActivateScriptSystemdSupervisor(t *testing.T) {
|
||||||
|
script := activateScript(activateParams{
|
||||||
|
Stage: "/opt/rsmon-worker-src/rsmon-worker",
|
||||||
|
Binary: "/usr/local/bin/rsmon-worker",
|
||||||
|
ConfigDir: "/etc/rsmon-worker",
|
||||||
|
EnvFile: "/etc/rsmon-worker/worker.env",
|
||||||
|
EnvTmp: "/tmp/e.env",
|
||||||
|
DataDir: "/var/lib/rsmon-worker",
|
||||||
|
UnitTmp: "/tmp/u.service",
|
||||||
|
UnitFile: "/etc/systemd/system/rsmon-worker.service",
|
||||||
|
UnitMode: "0644",
|
||||||
|
UnitName: "rsmon-worker.service",
|
||||||
|
RCName: "rsmon-worker",
|
||||||
|
Supervisor: "systemd",
|
||||||
|
NoStart: "0",
|
||||||
|
})
|
||||||
|
for _, want := range []string{
|
||||||
|
"supervisor=systemd",
|
||||||
|
"systemctl daemon-reload",
|
||||||
|
"systemctl restart \"$unit_name\"",
|
||||||
|
"systemctl is-active --quiet \"$unit_name\"",
|
||||||
|
"systemctl enable \"$unit_name\"",
|
||||||
|
"svc_active()",
|
||||||
|
"start failure: systemctl restart $unit_name failed",
|
||||||
|
"start failure: worker service is not active",
|
||||||
|
} {
|
||||||
|
if !strings.Contains(script, want) {
|
||||||
|
t.Fatalf("systemd supervisor missing %q:\n%s", want, script)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// The systemd health loop must use `systemctl is-active`, never the
|
||||||
|
// embedded supervisor's pid file.
|
||||||
|
if !strings.Contains(script, "systemd) systemctl is-active --quiet \"$unit_name\"") {
|
||||||
|
t.Fatalf("systemd health loop must use systemctl is-active:\n%s", script)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestActivateScriptOpenRCSupervisor(t *testing.T) {
|
||||||
|
script := activateScript(activateParams{
|
||||||
|
Stage: "/opt/rsmon-worker-src/rsmon-worker",
|
||||||
|
Binary: "/usr/local/bin/rsmon-worker",
|
||||||
|
ConfigDir: "/etc/rsmon-worker",
|
||||||
|
EnvFile: "/etc/rsmon-worker/worker.env",
|
||||||
|
EnvTmp: "/tmp/e.env",
|
||||||
|
DataDir: "/var/lib/rsmon-worker",
|
||||||
|
UnitTmp: "/tmp/u",
|
||||||
|
UnitFile: "/etc/init.d/rsmon-worker",
|
||||||
|
UnitMode: "0755",
|
||||||
|
UnitName: "rsmon-worker.service",
|
||||||
|
RCName: "rsmon-worker",
|
||||||
|
Supervisor: "openrc",
|
||||||
|
NoStart: "0",
|
||||||
|
})
|
||||||
|
for _, want := range []string{
|
||||||
|
"rc-service \"$rc_name\" restart",
|
||||||
|
"rc-service \"$rc_name\" status",
|
||||||
|
"rc-update add \"$rc_name\" default",
|
||||||
|
"start failure: rc-service restart $rc_name failed",
|
||||||
|
"rc-update del \"$rc_name\" default",
|
||||||
|
} {
|
||||||
|
if !strings.Contains(script, want) {
|
||||||
|
t.Fatalf("openrc supervisor missing %q:\n%s", want, script)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestActivateScriptNoStartSkipsRestart(t *testing.T) {
|
||||||
|
script := activateScript(activateParams{
|
||||||
|
Stage: "/opt/rsmon-worker-src/rsmon-worker", Binary: "/usr/local/bin/rsmon-worker",
|
||||||
|
ConfigDir: "/etc/rsmon-worker", EnvFile: "/etc/rsmon-worker/worker.env", EnvTmp: "/tmp/e.env",
|
||||||
|
DataDir: "/var/lib/rsmon-worker", UnitTmp: "", UnitFile: "", UnitMode: "",
|
||||||
|
UnitName: "rsmon-worker.service", RCName: "rsmon-worker", Supervisor: "none", NoStart: "1",
|
||||||
|
})
|
||||||
|
if !strings.Contains(script, "[ \"$no_start\" -ne 1 ]") {
|
||||||
|
t.Fatalf("no-start gate missing:\n%s", script)
|
||||||
|
}
|
||||||
|
if !strings.Contains(script, "started=no") {
|
||||||
|
t.Fatalf("no-start summary missing:\n%s", script)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestActivateScriptNeverContainsSecrets(t *testing.T) {
|
||||||
|
const secret = "super-secret-token-value"
|
||||||
|
params := activateParams{
|
||||||
|
Stage: "/opt/rsmon-worker-src/rsmon-worker", Binary: "/usr/local/bin/rsmon-worker",
|
||||||
|
ConfigDir: "/etc/rsmon-worker", EnvFile: "/etc/rsmon-worker/worker.env", EnvTmp: "/tmp/e.env",
|
||||||
|
DataDir: "/var/lib/rsmon-worker", UnitTmp: "/tmp/u", UnitFile: "/etc/systemd/system/rsmon-worker.service",
|
||||||
|
UnitMode: "0644", UnitName: "rsmon-worker.service", RCName: "rsmon-worker",
|
||||||
|
Supervisor: "systemd", NoStart: "0",
|
||||||
|
}
|
||||||
|
script := activateScript(params)
|
||||||
|
if strings.Contains(script, secret) {
|
||||||
|
t.Fatalf("activate script contains a secret:\n%s", script)
|
||||||
|
}
|
||||||
|
// The env file is referenced by path; its contents (the secrets) are
|
||||||
|
// only sourced at runtime and never echoed.
|
||||||
|
for _, want := range []string{"RSMON_TOKEN=", secret} {
|
||||||
|
if strings.Contains(script, want) {
|
||||||
|
t.Fatalf("activate script must not embed env contents (%q):\n%s", want, script)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestOpenRCInit(t *testing.T) {
|
||||||
|
unit := openrcInitFor(resolvePaths(""))
|
||||||
|
for _, want := range []string{
|
||||||
|
"#!/sbin/openrc-run",
|
||||||
|
"name=rsmon-worker",
|
||||||
|
"description=RSMon distributed monitoring worker",
|
||||||
|
"command=/usr/local/bin/rsmon-worker",
|
||||||
|
"command_background=true",
|
||||||
|
"pidfile=/var/lib/rsmon-worker/worker.pid",
|
||||||
|
"output_log=/var/lib/rsmon-worker/worker.log",
|
||||||
|
"need net",
|
||||||
|
". /etc/rsmon-worker/worker.env",
|
||||||
|
"set -a",
|
||||||
|
"set +a",
|
||||||
|
"RSMON_WEBAPP_DATA_DIR=/var/lib/rsmon-worker/webapp",
|
||||||
|
} {
|
||||||
|
if !strings.Contains(unit, want) {
|
||||||
|
t.Fatalf("openrc init missing %q:\n%s", want, unit)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
named := openrcInitFor(resolvePaths("edge"))
|
||||||
|
for _, want := range []string{
|
||||||
|
"name=rsmon-worker-edge", "command=/usr/local/bin/rsmon-worker-edge",
|
||||||
|
"pidfile=/var/lib/rsmon-worker-edge/worker.pid", "/etc/rsmon-worker-edge/worker.env", "(edge)",
|
||||||
|
} {
|
||||||
|
if !strings.Contains(named, want) {
|
||||||
|
t.Fatalf("named openrc init missing %q:\n%s", want, named)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunningInitScript(t *testing.T) {
|
||||||
|
for _, want := range []string{"set -eu", "/run/systemd/system", "systemctl", "/run/openrc/softlevel", "rc-service", "echo none"} {
|
||||||
|
if !strings.Contains(runningInitScript, want) {
|
||||||
|
t.Fatalf("running-init script missing %q:\n%s", want, runningInitScript)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNormalizeSupervisor(t *testing.T) {
|
||||||
|
for _, in := range []string{"systemd", "openrc", "systemd\n", " openrc "} {
|
||||||
|
want := strings.TrimSpace(in)
|
||||||
|
if got := normalizeSupervisor(in); got != want {
|
||||||
|
t.Fatalf("normalizeSupervisor(%q) = %q, want %q", in, got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, in := range []string{"", "none", "sysvinit", " "} {
|
||||||
|
if got := normalizeSupervisor(in); got != "none" {
|
||||||
|
t.Fatalf("normalizeSupervisor(%q) = %q, want none", in, got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestUnitContent(t *testing.T) {
|
||||||
|
p := resolvePaths("")
|
||||||
|
systemd, file, mode := unitContent(sshinstall.InitSystemd, p)
|
||||||
|
if systemd == "" || file != "/etc/systemd/system/rsmon-worker.service" || mode != "0644" {
|
||||||
|
t.Fatalf("systemd unit content = %q, %q, %q", systemd, file, mode)
|
||||||
|
}
|
||||||
|
if !strings.Contains(systemd, "ExecStart=/usr/local/bin/rsmon-worker\n") {
|
||||||
|
t.Fatalf("systemd unit not the classic hardened unit:\n%s", systemd)
|
||||||
|
}
|
||||||
|
openrc, file, mode := unitContent(sshinstall.InitOpenRC, p)
|
||||||
|
if openrc == "" || file != "/etc/init.d/rsmon-worker" || mode != "0755" {
|
||||||
|
t.Fatalf("openrc unit content = %q, %q, %q", openrc, file, mode)
|
||||||
|
}
|
||||||
|
if content, file, mode := unitContent(sshinstall.InitUnknown, p); content != "" || file != "" || mode != "" {
|
||||||
|
t.Fatalf("unknown-init unit content = %q, %q, %q, want empty (no-service gate)", content, file, mode)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRenderEnvActivation(t *testing.T) {
|
||||||
|
t.Setenv("RSMON_URL", "")
|
||||||
|
t.Setenv("RSMON_TOKEN", "")
|
||||||
|
t.Setenv("WORKER_HOST", "")
|
||||||
|
t.Setenv("WORKER_PORT", "")
|
||||||
|
t.Setenv("WORKER_LOGIN", "")
|
||||||
|
t.Setenv("WORKER_PASSWORD", "")
|
||||||
|
|
||||||
|
data, err := (ActivationOptions{Activate: true, URL: "https://rsmon.ru", Token: "secret"}).renderEnv()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, want := range []string{"RSMON_URL=https://rsmon.ru\n", "RSMON_TOKEN=secret\n", "WORKER_HOST=127.0.0.1\n", "WORKER_PORT=27401\n"} {
|
||||||
|
if !strings.Contains(string(data), want) {
|
||||||
|
t.Fatalf("rendered env missing %q:\n%s", want, data)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// PUBLIC_URL canonicalization is reused from the classic installer.
|
||||||
|
data, err = (ActivationOptions{
|
||||||
|
Activate: true, URL: "https://rsmon.ru", Token: "secret",
|
||||||
|
PublicURL: "https://worker.example.com",
|
||||||
|
}).renderEnv()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !strings.Contains(string(data), "PUBLIC_URL=https://worker.example.com\n") {
|
||||||
|
t.Fatalf("rendered env missing PUBLIC_URL:\n%s", data)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRenderEnvActivationRejectsMixedBasicAuth(t *testing.T) {
|
||||||
|
t.Setenv("RSMON_TOKEN", "")
|
||||||
|
if _, err := (ActivationOptions{
|
||||||
|
Activate: true, URL: "https://rsmon.ru", Token: "secret",
|
||||||
|
Login: "admin",
|
||||||
|
}).renderEnv(); err == nil {
|
||||||
|
t.Fatal("login-only basic auth accepted")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRenderEnvActivationRequiresToken(t *testing.T) {
|
||||||
|
if _, err := (ActivationOptions{Activate: true, URL: "https://rsmon.ru"}).renderEnv(); err == nil {
|
||||||
|
t.Fatal("activation without a token accepted")
|
||||||
|
}
|
||||||
|
if _, err := (ActivationOptions{Activate: true, Token: "x", URL: "not-a-url"}).renderEnv(); err == nil {
|
||||||
|
t.Fatal("activation with a malformed URL accepted")
|
||||||
|
}
|
||||||
|
if _, err := (ActivationOptions{Activate: true, Token: "x", URL: "https://rsmon.ru", Name: "Bad_Name"}).renderEnv(); err == nil {
|
||||||
|
t.Fatal("activation with an invalid instance name accepted")
|
||||||
|
}
|
||||||
|
if _, err := (ActivationOptions{Activate: true, Token: "x", URL: "https://rsmon.ru", Name: "edge"}).renderEnv(); err == nil {
|
||||||
|
t.Fatal("named activation without WORKER_PORT accepted")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestRenderEnvActivationReadsEnvFileOnce proves the env file is read a
|
||||||
|
// single time and validated from the in-memory bytes: swapping the file
|
||||||
|
// after the read cannot smuggle a different value into the render, and a
|
||||||
|
// malformed file fails on the read bytes.
|
||||||
|
func TestRenderEnvActivationReadsEnvFileOnce(t *testing.T) {
|
||||||
|
t.Setenv("RSMON_TOKEN", "")
|
||||||
|
path := filepath.Join(t.TempDir(), "worker.env")
|
||||||
|
if err := os.WriteFile(path, []byte("RSMON_URL=https://rsmon.ru\nRSMON_TOKEN=file-token\nWORKER_PORT=28888\n"), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
data, err := (ActivationOptions{Activate: true, EnvFile: path}).renderEnv()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !strings.Contains(string(data), "RSMON_TOKEN=file-token\n") || !strings.Contains(string(data), "WORKER_PORT=28888\n") {
|
||||||
|
t.Fatalf("env file values not rendered:\n%s", data)
|
||||||
|
}
|
||||||
|
// A malformed file must be rejected from the same read.
|
||||||
|
if err := os.WriteFile(path, []byte("RSMON_URL=https://rsmon.ru\nRSMON_TOKEN=has space\n"), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := (ActivationOptions{Activate: true, EnvFile: path}).renderEnv(); err == nil {
|
||||||
|
t.Fatal("malformed env file accepted")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestNormalizeSourceOptionsActivationRendersOnce verifies the rendered
|
||||||
|
// env is computed during normalization and reused, so a later call cannot
|
||||||
|
// re-read a changed env file.
|
||||||
|
func TestNormalizeSourceOptionsActivationRendersOnce(t *testing.T) {
|
||||||
|
t.Setenv("RSMON_TOKEN", "")
|
||||||
|
path := filepath.Join(t.TempDir(), "worker.env")
|
||||||
|
if err := os.WriteFile(path, []byte("RSMON_URL=https://rsmon.ru\nRSMON_TOKEN=once-token\n"), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
base := SourceInstallOptions{
|
||||||
|
SSHOptions: SSHOptions{Host: "h", User: "u"},
|
||||||
|
Activation: ActivationOptions{Activate: true, EnvFile: path},
|
||||||
|
}
|
||||||
|
norm, err := normalizeSourceOptions(base)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !strings.Contains(string(norm.activationEnv), "RSMON_TOKEN=once-token\n") {
|
||||||
|
t.Fatalf("activation env not cached during normalization: %q", norm.activationEnv)
|
||||||
|
}
|
||||||
|
// Even after the file changes, the cached render is authoritative.
|
||||||
|
if err := os.WriteFile(path, []byte("RSMON_URL=https://evil.test\nRSMON_TOKEN=evil\n"), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if strings.Contains(string(norm.activationEnv), "evil") {
|
||||||
|
t.Fatalf("changed env file leaked into the cached render: %q", norm.activationEnv)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNormalizeSourceOptionsActivation(t *testing.T) {
|
||||||
|
t.Setenv("RSMON_TOKEN", "")
|
||||||
|
base := SourceInstallOptions{SSHOptions: SSHOptions{Host: "h", User: "u"}}
|
||||||
|
if _, err := normalizeSourceOptions(base); err != nil {
|
||||||
|
t.Fatalf("staging-only options must stay valid: %v", err)
|
||||||
|
}
|
||||||
|
act := base
|
||||||
|
act.Activation = ActivationOptions{Activate: true, URL: "https://rsmon.ru"}
|
||||||
|
if _, err := normalizeSourceOptions(act); err == nil || !strings.Contains(err.Error(), "RSMON_TOKEN") {
|
||||||
|
t.Fatalf("activation without a token error = %v", err)
|
||||||
|
}
|
||||||
|
ok := base
|
||||||
|
ok.Activation = ActivationOptions{Activate: true, URL: "https://rsmon.ru", Token: "secret"}
|
||||||
|
if _, err := normalizeSourceOptions(ok); err != nil {
|
||||||
|
t.Fatalf("valid activation rejected: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestValidateRenderedEnv(t *testing.T) {
|
||||||
|
if err := validateRenderedEnv([]byte("RSMON_URL=https://rsmon.ru\nRSMON_TOKEN=secret\n")); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := validateRenderedEnv([]byte("RSMON_TOKEN=secret\n")); err == nil {
|
||||||
|
t.Fatal("env without RSMON_URL accepted")
|
||||||
|
}
|
||||||
|
if err := validateRenderedEnv([]byte("RSMON_URL=https://rsmon.ru\nRSMON_TOKEN=secret value\n")); err == nil {
|
||||||
|
t.Fatal("env with whitespace accepted")
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -50,11 +50,23 @@ type SourceInstallOptions struct {
|
|||||||
ToolchainDir string
|
ToolchainDir string
|
||||||
// StageBinary is where the built worker binary is written. It must
|
// StageBinary is where the built worker binary is written. It must
|
||||||
// be absolute and defaults to <BuildDir>/rsmon-worker. The running
|
// be absolute and defaults to <BuildDir>/rsmon-worker. The running
|
||||||
// service and its config are NOT touched by this work package.
|
// service and its config are NOT touched until Activation runs.
|
||||||
StageBinary string
|
StageBinary string
|
||||||
// SessionTimeout bounds each remote command. 0 uses the default
|
// SessionTimeout bounds each remote command. 0 uses the default
|
||||||
// (30 minutes); the build step can legitimately run for minutes.
|
// (30 minutes); the build step can legitimately run for minutes.
|
||||||
SessionTimeout time.Duration
|
SessionTimeout time.Duration
|
||||||
|
// Activation drives work package 4: after the staging build, the
|
||||||
|
// staged binary, validated environment, data dir, and the detected
|
||||||
|
// init's service definition are installed atomically and the worker
|
||||||
|
// is started and verified (process + /healthz). Any failure rolls
|
||||||
|
// back to the prior working install. Empty keeps SourceInstall at the
|
||||||
|
// staging boundary and touches no service configuration.
|
||||||
|
Activation ActivationOptions
|
||||||
|
|
||||||
|
// activationEnv is the once-rendered activation environment, computed
|
||||||
|
// during option normalization so the env file is read and rendered
|
||||||
|
// exactly once per run (no TOCTOU between preflight and activation).
|
||||||
|
activationEnv []byte
|
||||||
}
|
}
|
||||||
|
|
||||||
// SourceInstallResult is what a source installation resolved to. The
|
// SourceInstallResult is what a source installation resolved to. The
|
||||||
@@ -70,6 +82,10 @@ type SourceInstallResult struct {
|
|||||||
ResolvedCommit string
|
ResolvedCommit string
|
||||||
RecordFile string
|
RecordFile string
|
||||||
StageBinary string
|
StageBinary string
|
||||||
|
// Activation is set when the staged binary was atomically installed
|
||||||
|
// and the worker started/verified. It records the installed layout
|
||||||
|
// and the supervisor used. Nil when Activation was not requested.
|
||||||
|
Activation *ActivationResult
|
||||||
}
|
}
|
||||||
|
|
||||||
// defaultToolchainDir is the standard Go installation prefix.
|
// defaultToolchainDir is the standard Go installation prefix.
|
||||||
@@ -142,14 +158,20 @@ func (e *sourceExecutor) fileProber() sshinstall.FileProber {
|
|||||||
// branch, builds the worker to a staging path, and only then records the
|
// branch, builds the worker to a staging path, and only then records the
|
||||||
// resolved branch and commit.
|
// resolved branch and commit.
|
||||||
//
|
//
|
||||||
// The running service, its configuration, and its data directory are
|
// When Activation.Activate is set, the staged binary is then atomically
|
||||||
// deliberately untouched: atomic activation and rollback are the next
|
// installed together with the validated environment, data directory, and
|
||||||
// work package. Every remote step runs with the same privilege path as
|
// the detected init's service definition, and the worker is started and
|
||||||
// `deploy` (root, passwordless sudo, or sudo -S), every interpolated
|
// verified (process + /healthz); any activation/start/health failure
|
||||||
// value is single-quoted, every step script fails closed (`set -eu` or
|
// rolls back to the prior working install. Without activation the
|
||||||
// explicit `&&`/retry), and every failure returns a bounded, actionable
|
// running service, its configuration, and its data directory are
|
||||||
// error. Each remote command is capped by SessionTimeout and its stdout
|
// deliberately untouched (the staging boundary of work package 3).
|
||||||
// is size-bounded.
|
//
|
||||||
|
// Every remote step runs with the same privilege path as `deploy` (root,
|
||||||
|
// passwordless sudo, or sudo -S), every interpolated value is
|
||||||
|
// single-quoted, every step script fails closed (`set -eu` or explicit
|
||||||
|
// `&&`/retry), and every failure returns a bounded, actionable error.
|
||||||
|
// Each remote command is capped by SessionTimeout and its stdout is
|
||||||
|
// size-bounded.
|
||||||
func SourceInstall(opts SourceInstallOptions) (*SourceInstallResult, error) {
|
func SourceInstall(opts SourceInstallOptions) (*SourceInstallResult, error) {
|
||||||
opts, err := normalizeSourceOptions(opts)
|
opts, err := normalizeSourceOptions(opts)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -276,6 +298,15 @@ func SourceInstall(opts SourceInstallOptions) (*SourceInstallResult, error) {
|
|||||||
if _, err := executor.runPrivileged("sh -c " + shellQuote(commitRecordScript(plan.BuildDir, branch, commit))); err != nil {
|
if _, err := executor.runPrivileged("sh -c " + shellQuote(commitRecordScript(plan.BuildDir, branch, commit))); err != nil {
|
||||||
return nil, fmt.Errorf("record resolved commit: %w", err)
|
return nil, fmt.Errorf("record resolved commit: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Work package 4: atomically install the staged build and activate
|
||||||
|
// the worker. On any failure the remote script restores the previous
|
||||||
|
// working install and this step returns a bounded error.
|
||||||
|
if opts.Activation.Activate {
|
||||||
|
if err := executor.activateWorker(opts, result); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
}
|
||||||
return result, nil
|
return result, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -316,6 +347,20 @@ func normalizeSourceOptions(o SourceInstallOptions) (SourceInstallOptions, error
|
|||||||
if o.SessionTimeout <= 0 {
|
if o.SessionTimeout <= 0 {
|
||||||
o.SessionTimeout = defaultSessionTimeout
|
o.SessionTimeout = defaultSessionTimeout
|
||||||
}
|
}
|
||||||
|
if o.Activation.Activate {
|
||||||
|
// Render (and validate) the activation environment exactly once
|
||||||
|
// here. The rendered bytes are reused at activation time, so the
|
||||||
|
// env file is read a single time and cannot change between the
|
||||||
|
// preflight and the remote install (env-file TOCTOU).
|
||||||
|
env, err := o.Activation.renderEnv()
|
||||||
|
if err != nil {
|
||||||
|
return o, fmt.Errorf("activation: %w", err)
|
||||||
|
}
|
||||||
|
if err := validateRenderedEnv(env); err != nil {
|
||||||
|
return o, fmt.Errorf("activation: %w", err)
|
||||||
|
}
|
||||||
|
o.activationEnv = env
|
||||||
|
}
|
||||||
return o, nil
|
return o, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ import (
|
|||||||
"crypto/ed25519"
|
"crypto/ed25519"
|
||||||
"crypto/rand"
|
"crypto/rand"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"io"
|
||||||
"net"
|
"net"
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
@@ -106,7 +107,18 @@ func (s *fakeSSHServer) handleConn(conn net.Conn, config *ssh.ServerConfig) {
|
|||||||
}
|
}
|
||||||
go func() {
|
go func() {
|
||||||
defer channel.Close()
|
defer channel.Close()
|
||||||
|
// Drain the client's stdin so uploads (which stream base64
|
||||||
|
// over the session stdin) never block the channel window.
|
||||||
|
// The channel is not closed until the client's write side is
|
||||||
|
// exhausted, mirroring the real server behavior.
|
||||||
|
var drained sync.WaitGroup
|
||||||
|
drained.Add(1)
|
||||||
|
go func() {
|
||||||
|
defer drained.Done()
|
||||||
|
_, _ = io.Copy(io.Discard, channel)
|
||||||
|
}()
|
||||||
s.handleSession(channel, requests)
|
s.handleSession(channel, requests)
|
||||||
|
drained.Wait()
|
||||||
}()
|
}()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -151,9 +163,10 @@ func (s *fakeSSHServer) execCommand(channel ssh.Channel, command string) {
|
|||||||
|
|
||||||
// defaultFakeExec simulates a minimal Linux host: it answers os-release,
|
// defaultFakeExec simulates a minimal Linux host: it answers os-release,
|
||||||
// uname, the init-marker file probe, origin URL, branch resolution, and
|
// uname, the init-marker file probe, origin URL, branch resolution, and
|
||||||
// rev-parse, and accepts every install step. A pinned-branch existence
|
// rev-parse, accepts every install step, and simulates the work-package-4
|
||||||
// check (show-ref) fails by default so the missing-branch path is
|
// activation (running-init probe, env/unit uploads, and the activate
|
||||||
// exercised without extra setup.
|
// script succeeding). A pinned-branch existence check (show-ref) fails by
|
||||||
|
// default so the missing-branch path is exercised without extra setup.
|
||||||
func defaultFakeExec(command string) (string, string, int) {
|
func defaultFakeExec(command string) (string, string, int) {
|
||||||
switch {
|
switch {
|
||||||
case strings.Contains(command, "cat /etc/os-release"):
|
case strings.Contains(command, "cat /etc/os-release"):
|
||||||
@@ -170,6 +183,18 @@ func defaultFakeExec(command string) (string, string, int) {
|
|||||||
return fakeCommitHex + "\n", "", 0
|
return fakeCommitHex + "\n", "", 0
|
||||||
case strings.Contains(command, "show-ref"):
|
case strings.Contains(command, "show-ref"):
|
||||||
return "", "branch not found", 1
|
return "", "branch not found", 1
|
||||||
|
case strings.Contains(command, "softlevel"):
|
||||||
|
// running-init probe: no systemd/openrc is actually booted.
|
||||||
|
return "none\n", "", 0
|
||||||
|
case strings.Contains(command, "mktemp -d /tmp/rsmon-worker-act"):
|
||||||
|
// server-side secure 0700 upload dir.
|
||||||
|
return "/tmp/rsmon-worker-act-fake\n", "", 0
|
||||||
|
case strings.Contains(command, "base64 -d"):
|
||||||
|
// env/unit uploads succeed.
|
||||||
|
return "", "", 0
|
||||||
|
case strings.Contains(command, "rsmon-worker activated"):
|
||||||
|
// the activate script completed successfully.
|
||||||
|
return "", "", 0
|
||||||
default:
|
default:
|
||||||
return "", "", 0
|
return "", "", 0
|
||||||
}
|
}
|
||||||
@@ -184,9 +209,25 @@ func testSSHOptions(port int) SourceInstallOptions {
|
|||||||
Password: fakeSSHPassword,
|
Password: fakeSSHPassword,
|
||||||
InsecureHostKey: true,
|
InsecureHostKey: true,
|
||||||
},
|
},
|
||||||
|
// Staging-only by default so the work-package-3 orchestration
|
||||||
|
// tests keep their historical shape; activation tests opt in.
|
||||||
|
Activation: ActivationOptions{Activate: false},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// testActivationOptions wraps testSSHOptions with the credentials and
|
||||||
|
// activation flag needed to run the work-package-4 flow against the fake
|
||||||
|
// server.
|
||||||
|
func testActivationOptions(port int) SourceInstallOptions {
|
||||||
|
opts := testSSHOptions(port)
|
||||||
|
opts.Activation = ActivationOptions{
|
||||||
|
Activate: true,
|
||||||
|
URL: "https://rsmon.ru",
|
||||||
|
Token: fakeSSHPassword + "-token",
|
||||||
|
}
|
||||||
|
return opts
|
||||||
|
}
|
||||||
|
|
||||||
func TestSourceInstallSSHFlow(t *testing.T) {
|
func TestSourceInstallSSHFlow(t *testing.T) {
|
||||||
srv := startFakeSSHServer(t, nil)
|
srv := startFakeSSHServer(t, nil)
|
||||||
res, err := SourceInstall(testSSHOptions(srv.Port()))
|
res, err := SourceInstall(testSSHOptions(srv.Port()))
|
||||||
@@ -349,3 +390,147 @@ func TestSourceInstallSSHCommandTimeout(t *testing.T) {
|
|||||||
t.Fatalf("err = %v, want command timeout", err)
|
t.Fatalf("err = %v, want command timeout", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestSourceInstallSSHActivationFlow verifies the work-package-4 flow
|
||||||
|
// over a real SSH session: after the staging build and commit record, the
|
||||||
|
// installer probes the running init, uploads the env and service
|
||||||
|
// definition, runs the atomic activation script, and reports the installed
|
||||||
|
// layout. The secrets-absent contract holds: the worker token never
|
||||||
|
// reaches a remote command.
|
||||||
|
func TestSourceInstallSSHActivationFlow(t *testing.T) {
|
||||||
|
srv := startFakeSSHServer(t, nil)
|
||||||
|
res, err := SourceInstall(testActivationOptions(srv.Port()))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if res.Activation == nil {
|
||||||
|
t.Fatal("activation result missing")
|
||||||
|
}
|
||||||
|
if res.Activation.Binary != "/usr/local/bin/rsmon-worker" ||
|
||||||
|
res.Activation.EnvFile != "/etc/rsmon-worker/worker.env" ||
|
||||||
|
res.Activation.DataDir != "/var/lib/rsmon-worker" ||
|
||||||
|
res.Activation.UnitFile != "/etc/systemd/system/rsmon-worker.service" ||
|
||||||
|
res.Activation.Supervisor != "none" ||
|
||||||
|
!res.Activation.Started {
|
||||||
|
t.Fatalf("activation result = %+v", res.Activation)
|
||||||
|
}
|
||||||
|
|
||||||
|
commands := srv.Commands()
|
||||||
|
var joined strings.Builder
|
||||||
|
for _, c := range commands {
|
||||||
|
joined.WriteString(c)
|
||||||
|
joined.WriteString("\n")
|
||||||
|
}
|
||||||
|
for _, want := range []string{
|
||||||
|
"/run/systemd/system",
|
||||||
|
"softlevel",
|
||||||
|
"mktemp -d /tmp/rsmon-worker-act",
|
||||||
|
"umask 077; base64 -d >",
|
||||||
|
"rsmon-worker activated",
|
||||||
|
"install -m 0755",
|
||||||
|
"install -m 0600",
|
||||||
|
"chmod 0750",
|
||||||
|
"worker.pid",
|
||||||
|
"liveness",
|
||||||
|
"rm -rf --",
|
||||||
|
} {
|
||||||
|
if !strings.Contains(joined.String(), want) {
|
||||||
|
t.Fatalf("activation commands missing %q:\n%s", want, joined.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Ordering: build and commit record, then a server-side secure upload
|
||||||
|
// dir is created, the env/unit are uploaded into it, and the activation
|
||||||
|
// script runs last.
|
||||||
|
idx := func(sub string) int {
|
||||||
|
for i, c := range commands {
|
||||||
|
if strings.Contains(c, sub) {
|
||||||
|
return i
|
||||||
|
}
|
||||||
|
}
|
||||||
|
t.Fatalf("command %q not found in %v", sub, commands)
|
||||||
|
return -1
|
||||||
|
}
|
||||||
|
if idx("branch=%s") >= idx("mktemp -d /tmp/rsmon-worker-act") ||
|
||||||
|
idx("mktemp -d /tmp/rsmon-worker-act") >= idx("base64 -d >") ||
|
||||||
|
idx("base64 -d >") >= idx("rsmon-worker activated") {
|
||||||
|
t.Fatalf("activation order wrong: %v", commands)
|
||||||
|
}
|
||||||
|
if strings.Contains(joined.String(), fakeSSHPassword+"-token") {
|
||||||
|
t.Fatal("worker token leaked into a remote command")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestSourceInstallSSHNoActivationSkipsService proves the staging
|
||||||
|
// boundary: without Activation the installer never probes the init system,
|
||||||
|
// uploads an env/unit, or runs the activation script.
|
||||||
|
func TestSourceInstallSSHNoActivationSkipsService(t *testing.T) {
|
||||||
|
srv := startFakeSSHServer(t, nil)
|
||||||
|
res, err := SourceInstall(testSSHOptions(srv.Port()))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if res.Activation != nil {
|
||||||
|
t.Fatalf("activation must not run without the flag: %+v", res.Activation)
|
||||||
|
}
|
||||||
|
joined := strings.Join(srv.Commands(), "\n")
|
||||||
|
for _, forbidden := range []string{"softlevel", "base64 -d >", "rsmon-worker activated", "worker.pid"} {
|
||||||
|
if strings.Contains(joined, forbidden) {
|
||||||
|
t.Fatalf("staging-only flow ran activation step %q:\n%s", forbidden, joined)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestSourceInstallSSHActivationRequiresToken verifies the credentials
|
||||||
|
// gate: activation without a token fails before any remote connection.
|
||||||
|
func TestSourceInstallSSHActivationRequiresToken(t *testing.T) {
|
||||||
|
srv := startFakeSSHServer(t, nil)
|
||||||
|
opts := testSSHOptions(srv.Port())
|
||||||
|
opts.Activation = ActivationOptions{Activate: true, URL: "https://rsmon.ru"}
|
||||||
|
_, err := SourceInstall(opts)
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "RSMON_TOKEN") {
|
||||||
|
t.Fatalf("err = %v, want token requirement", err)
|
||||||
|
}
|
||||||
|
if got := len(srv.Commands()); got != 0 {
|
||||||
|
t.Fatalf("commands ran before validation failed: %d", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestSourceInstallSSHActivationFailure verifies the bounded failure
|
||||||
|
// path: a failing activate script surfaces as a labelled error with the
|
||||||
|
// remote stderr, and the uploaded temp env/unit are cleaned up by the
|
||||||
|
// controller defer regardless of the outcome.
|
||||||
|
func TestSourceInstallSSHActivationFailure(t *testing.T) {
|
||||||
|
srv := startFakeSSHServer(t, func(command string) (string, string, int) {
|
||||||
|
if strings.Contains(command, "rsmon-worker activated") {
|
||||||
|
return "", "activation exploded: disk full", 1
|
||||||
|
}
|
||||||
|
return defaultFakeExec(command)
|
||||||
|
})
|
||||||
|
_, err := SourceInstall(testActivationOptions(srv.Port()))
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "activate worker service") || !strings.Contains(err.Error(), "disk full") {
|
||||||
|
t.Fatalf("err = %v, want bounded activation failure", err)
|
||||||
|
}
|
||||||
|
joined := strings.Join(srv.Commands(), "\n")
|
||||||
|
if !strings.Contains(joined, "rm -rf --") {
|
||||||
|
t.Fatalf("secure upload dir cleanup not issued after activation failure:\n%s", joined)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestSourceInstallSSHActivationHealthFailure verifies the health gate
|
||||||
|
// surfaces as an activation failure and the temp files are still cleaned.
|
||||||
|
// The fake cannot run the shell script, so the failure is injected at the
|
||||||
|
// activate-command level; the real rollback semantics are covered by the
|
||||||
|
// Docker/OpenSSH E2E tests.
|
||||||
|
func TestSourceInstallSSHActivationHealthFailure(t *testing.T) {
|
||||||
|
srv := startFakeSSHServer(t, func(command string) (string, string, int) {
|
||||||
|
if strings.Contains(command, "rsmon-worker activated") {
|
||||||
|
return "", "health failure: worker did not answer /healthz", 1
|
||||||
|
}
|
||||||
|
return defaultFakeExec(command)
|
||||||
|
})
|
||||||
|
_, err := SourceInstall(testActivationOptions(srv.Port()))
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "did not answer /healthz") {
|
||||||
|
t.Fatalf("err = %v, want /healthz verification failure", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
Ссылка в новой задаче
Block a user