Сравнить коммиты
5 Коммитов
cb23f123ae
...
master
| Автор | SHA1 | Дата | |
|---|---|---|---|
|
|
b8c7596fc5 | ||
|
|
714dda08e5 | ||
|
|
674a7d82bf | ||
|
|
bd6070ee1f | ||
|
|
4651deb280 |
94
.github/workflows/test-ssh.yml
поставляемый
Обычный файл
94
.github/workflows/test-ssh.yml
поставляемый
Обычный файл
@@ -0,0 +1,94 @@
|
||||
name: SSH Source-Install E2E
|
||||
|
||||
# Work package 5 of docs/source-installation.md: run the Alpine/Ubuntu/Arch
|
||||
# OpenSSH source-install E2E matrix (make test-ssh) in CI.
|
||||
#
|
||||
# Trust boundary. This job builds and runs privileged Docker containers from
|
||||
# repository code (the distro fixtures execute the checked-out source), so it
|
||||
# runs ONLY on trusted refs: pushes to the default branch (master) and manual
|
||||
# workflow_dispatch. It deliberately does NOT trigger on pull_request: a PR
|
||||
# can carry untrusted code into a privileged Docker environment, and Gitea's
|
||||
# fork-PR read-only token clamp does not change what the containers can do on
|
||||
# the runner host. The ordinary unit `make test` CI run (ci.yml) stays
|
||||
# Docker-free and still covers pull requests.
|
||||
#
|
||||
# Runner requirement. The job inherits whatever Docker access `docker.yml`
|
||||
# already relies on. The harness dials fixture SSH ports published on the
|
||||
# Docker daemon loopback, so the runner must expose Docker with loopback port
|
||||
# publishing reachable from the job; scripts/ci/test-ssh.sh fails fast when
|
||||
# that is not the case.
|
||||
#
|
||||
# External network: fixture images (alpine from the reg.rsxx.ru mirror,
|
||||
# ubuntu/arch overridable via RSMON_TEST_IMAGE_*), the pinned Go toolchain
|
||||
# from go.dev, the public source repo from rocketgit.ru, and distro package
|
||||
# repos are fetched live. Operators can pin a resolver for flaky CI DNS via
|
||||
# the RSMON_TEST_DOCKER_DNS repository variable (comma-separated nameservers).
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- master
|
||||
workflow_dispatch:
|
||||
|
||||
# Per-ref scoping: master pushes cancel a superseded in-flight run on the same
|
||||
# ref instead of stacking; a manual dispatch on another branch has its own
|
||||
# group and never cancels the master run.
|
||||
concurrency:
|
||||
group: test-ssh-${{ gitea.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
# Least privilege: the job only needs to read the repository (actions/checkout
|
||||
# and the harness's SSH-free operations). contents:read is supported by Gitea
|
||||
# Actions (the GITEA_TOKEN scope for code/releases).
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
test-ssh:
|
||||
name: Alpine/Ubuntu/Arch source-install E2E
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 90
|
||||
env:
|
||||
# All empty by default (harness defaults / Docker embedded DNS).
|
||||
# Override in repo/organization variables:
|
||||
# RSMON_TEST_IMAGE_ALPINE / _UBUNTU / _ARCH - pin a mirror or a
|
||||
# specific distro snapshot for the fixture base image;
|
||||
# RSMON_TEST_DOCKER_DNS - comma-separated nameservers applied as
|
||||
# `docker run --dns ...` for flaky CI resolvers.
|
||||
RSMON_TEST_IMAGE_ALPINE: ${{ vars.RSMON_TEST_IMAGE_ALPINE }}
|
||||
RSMON_TEST_IMAGE_UBUNTU: ${{ vars.RSMON_TEST_IMAGE_UBUNTU }}
|
||||
RSMON_TEST_IMAGE_ARCH: ${{ vars.RSMON_TEST_IMAGE_ARCH }}
|
||||
RSMON_TEST_DOCKER_DNS: ${{ vars.RSMON_TEST_DOCKER_DNS }}
|
||||
steps:
|
||||
# Actions are pinned to immutable full commit SHAs (not moving tags).
|
||||
# Verified 2026-08-13 against the GitHub API that the commit the tag
|
||||
# points to is a commit object:
|
||||
# actions/checkout@v4 -> 11d5960a326750d5838078e36cf38b85af677262
|
||||
# actions/setup-go@v5 -> 40f1582b2485089dde7abd97c1529aa768e1baff
|
||||
# Other workflows (ci.yml, docker.yml) still use moving tags; see the
|
||||
# repo-wide convention note in docs/source-installation.md.
|
||||
- name: Check out code
|
||||
uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff
|
||||
with:
|
||||
go-version: '1.26.x'
|
||||
# Caches the Go module/build cache used to compile the harness test
|
||||
# binary. Safe: only public dependencies, no credentials.
|
||||
cache: true
|
||||
|
||||
- name: Run Alpine/Ubuntu/Arch OpenSSH source-install E2E
|
||||
run: bash scripts/ci/test-ssh.sh
|
||||
|
||||
# Belt-and-suspenders: the harness and test-ssh.sh already tear down
|
||||
# everything they create; this guarantees a killed job leaves no
|
||||
# rsmon-worker-test-* resources behind on the runner. Filters are
|
||||
# anchored to the harness's own prefix and image repository so cleanup
|
||||
# never touches a shared base image or unrelated resources.
|
||||
- name: Clean up leftover harness resources
|
||||
if: always()
|
||||
run: |
|
||||
docker ps -aq --filter "name=^rsmon-worker-test-" 2>/dev/null | xargs -r docker rm -f >/dev/null 2>&1 || true
|
||||
docker network ls -q --filter "name=^rsmon-worker-test-" 2>/dev/null | xargs -r docker network rm >/dev/null 2>&1 || true
|
||||
docker images -q --filter "reference=rsmon-worker-test/*" 2>/dev/null | xargs -r docker image rm -f >/dev/null 2>&1 || true
|
||||
20
Makefile
20
Makefile
@@ -4,17 +4,22 @@ COMMIT ?= $(shell git rev-parse --short=12 HEAD 2>/dev/null || printf unknown)
|
||||
BUILD_DATE ?= $(shell date -u +%Y-%m-%dT%H:%M:%SZ)
|
||||
LDFLAGS := -s -w -X main.version=$(VERSION) -X main.commit=$(COMMIT) -X main.buildDate=$(BUILD_DATE)
|
||||
|
||||
.PHONY: build test check image clean
|
||||
.PHONY: build test test-ssh check image clean
|
||||
|
||||
build:
|
||||
mkdir -p bin
|
||||
CGO_ENABLED=0 go build -trimpath -ldflags="$(LDFLAGS)" -o $(BINARY) ./cmd/rsmon-worker
|
||||
|
||||
test:
|
||||
RSMON_ENV=test CWD=$(CURDIR) go test \
|
||||
# RSMON_TEST_DOCKER=0 pins this target to the non-Docker unit run even
|
||||
# if a developer has the opt-in flag exported in their environment;
|
||||
# only the explicit `make test-ssh` target starts containers.
|
||||
RSMON_ENV=test CWD=$(CURDIR) RSMON_TEST_DOCKER=0 go test \
|
||||
./cmd/rsmon-worker \
|
||||
./internal/distworker \
|
||||
./internal/installer \
|
||||
./internal/installer/harness \
|
||||
./internal/sshinstall \
|
||||
./internal/webapp \
|
||||
./internal/compose \
|
||||
./internal/workercluster \
|
||||
@@ -33,6 +38,17 @@ test:
|
||||
./checks/cwhois \
|
||||
./checks/llmhttp
|
||||
|
||||
# test-ssh runs the Docker/OpenSSH source-install harness tests against
|
||||
# real Alpine, Ubuntu, and Arch fixtures, including the full work
|
||||
# package-3 source install (Go toolchain download, clone, and build) on
|
||||
# each. Requires a working Docker daemon; the fixtures pull/start
|
||||
# containers and build inside them, so this is opt-in and is never part
|
||||
# of the default `make test` run. The 60m timeout covers three real
|
||||
# toolchain downloads and worker builds plus the harness lifecycle tests.
|
||||
test-ssh:
|
||||
RSMON_ENV=test CWD=$(CURDIR) RSMON_TEST_DOCKER=1 go test \
|
||||
-v -timeout 60m -count=1 ./internal/installer/harness
|
||||
|
||||
check:
|
||||
go mod tidy
|
||||
git diff --exit-code -- go.mod go.sum
|
||||
|
||||
59
README.md
59
README.md
@@ -168,15 +168,68 @@ are rejected before connecting to the remote host.
|
||||
|
||||
The default SSH port is 22 and the default RSMon URL is `https://rsmon.ru`.
|
||||
Encrypted keys use `--key-passphrase-file`; password authentication uses
|
||||
`--password-file`; password-protected sudo uses `--sudo-password-file`. Direct
|
||||
secret flags are supported for interactive convenience but file options are
|
||||
safer for automation.
|
||||
`--password-file`; password-protected sudo uses `--sudo-password-file`. Strongly
|
||||
prefer the file options for automation: a secret supplied through a direct flag
|
||||
is visible in the process list and shell history, while a file option never
|
||||
exposes it through argv. The direct flags remain available for interactive
|
||||
convenience.
|
||||
|
||||
SSH host keys are checked against `~/.ssh/known_hosts` by default. Use
|
||||
`--known-hosts PATH` or pin `--host-key-fingerprint SHA256:...`. The explicit
|
||||
`--insecure-host-key` option disables host authentication and should only be
|
||||
used in a trusted disposable environment.
|
||||
|
||||
A Go SSH source installer is implemented as the `source-install` subcommand: it
|
||||
detects the remote distro and architecture over the existing SSH transport,
|
||||
installs the minimal build prerequisites, downloads and SHA-256-verifies the
|
||||
pinned Go 1.26 toolchain, clones/updates the public repository, records the
|
||||
resolved branch and commit, builds the worker to a staging path, and then
|
||||
atomically activates the staged binary together with the validated environment,
|
||||
data directory, and the detected init's service definition. It starts the
|
||||
worker and verifies the process and `/healthz`; any activation, start, or
|
||||
health failure rolls back to the previous working install (preserving the
|
||||
prior binary/env/unit and its enable state), concurrent activations are
|
||||
serialized by a lock, a run killed mid-flight is recovered from its
|
||||
leftover backup marker on the next activation, and reruns are idempotent
|
||||
(exactly one worker process, no leaked temp files). Pass `--no-activate`
|
||||
to stop at the staging build, and `--no-start` to install the layout
|
||||
without starting the worker. The pure detection/planning layer and the
|
||||
Docker/OpenSSH test harness that accepts it are implemented; see
|
||||
[`docs/source-installation.md`](docs/source-installation.md); run the live
|
||||
fixture matrix with `make test-ssh`.
|
||||
|
||||
```bash
|
||||
./bin/rsmon-worker source-install \
|
||||
--host worker.example.com \
|
||||
--user deploy \
|
||||
--identity-file ~/.ssh/id_ed25519 \
|
||||
--token-file ./worker.token
|
||||
```
|
||||
|
||||
By default the installer builds the remote's default branch and records what it
|
||||
resolves to (the public repository currently publishes `master`). Pass
|
||||
`--branch <name>` to pin an explicit branch; it must exist on the remote or the
|
||||
install fails before building. The repository must be an `https://` URL without
|
||||
userinfo.
|
||||
|
||||
The built binary is staged at `/opt/rsmon-worker-src/rsmon-worker` (override with
|
||||
`--build-dir` / `--stage-binary`), the toolchain at `/usr/local/go` (replaced
|
||||
atomically: download, verify, stage, swap with rollback), and the resolved
|
||||
branch and commit in `/opt/rsmon-worker-src/rsmon-worker.commit` (written only
|
||||
after a successful build). Activation installs the binary to
|
||||
`/usr/local/bin/rsmon-worker`, the environment to
|
||||
`/etc/rsmon-worker/worker.env` (mode 0600), the data directory to
|
||||
`/var/lib/rsmon-worker`, and a systemd unit or OpenRC init script for the
|
||||
detected init (an embedded supervisor manages the process when no init is
|
||||
running, as in containers). The env and unit are uploaded into a server-side
|
||||
0700 `mktemp -d` directory so no local user can race a predictable `/tmp` path,
|
||||
and the env file is read/rendered exactly once. The same SSH auth,
|
||||
secret-file, and host-key options as `deploy` apply; the worker token is
|
||||
supplied with `--token`/`--token-file` and written only to the mode-0600 env
|
||||
file. Prefer `--key-passphrase-file`, `--password-file`, and
|
||||
`--sudo-password-file` over their direct-flag equivalents: file options keep
|
||||
secrets out of the process list and shell history.
|
||||
|
||||
## Configuration
|
||||
|
||||
| Variable | Required | Default | Purpose |
|
||||
|
||||
@@ -18,6 +18,15 @@ func TestDispatchManagementCommand(t *testing.T) {
|
||||
if handled, code := dispatchManagementCommand([]string{"install", "--help"}); !handled || code != 0 {
|
||||
t.Fatalf("install help = handled %t code %d", handled, code)
|
||||
}
|
||||
if handled, code := dispatchManagementCommand([]string{"deploy", "--help"}); !handled || code != 0 {
|
||||
t.Fatalf("deploy help = handled %t code %d", handled, code)
|
||||
}
|
||||
if handled, code := dispatchManagementCommand([]string{"source-install", "--help"}); !handled || code != 0 {
|
||||
t.Fatalf("source-install help = handled %t code %d", handled, code)
|
||||
}
|
||||
if handled, code := dispatchManagementCommand([]string{"source-install", "--identity-file", t.TempDir() + "/missing"}); !handled || code == 0 {
|
||||
t.Fatalf("source-install without host/user = handled %t code %d", handled, code)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSecretValue(t *testing.T) {
|
||||
|
||||
@@ -19,6 +19,8 @@ func dispatchManagementCommand(args []string) (bool, int) {
|
||||
return true, installCommand(args[1:])
|
||||
case "deploy":
|
||||
return true, deployCommand(args[1:])
|
||||
case "source-install":
|
||||
return true, sourceInstallCommand(args[1:])
|
||||
default:
|
||||
return false, 0
|
||||
}
|
||||
@@ -153,3 +155,122 @@ func secretValue(direct, path string) (string, error) {
|
||||
}
|
||||
return strings.TrimRight(string(b), "\r\n"), nil
|
||||
}
|
||||
|
||||
// sourceInstallCommand drives the remote source installer
|
||||
// (docs/source-installation.md): prerequisites, verified Go toolchain,
|
||||
// clone/update, resolved branch/commit record, a staging build, and (work
|
||||
// package 4) the atomic activation of the staged binary, validated
|
||||
// environment, data dir, and the detected init's service definition,
|
||||
// followed by a start and a process + /healthz verification. It reuses
|
||||
// the deploy SSH options. Any activation/start/health failure rolls back
|
||||
// to the prior working install; --no-activate keeps the staging-only
|
||||
// behavior.
|
||||
func sourceInstallCommand(args []string) int {
|
||||
fs := flag.NewFlagSet("source-install", flag.ContinueOnError)
|
||||
fs.SetOutput(os.Stderr)
|
||||
var opts installer.SourceInstallOptions
|
||||
var passphraseFile, passwordFile, sudoPasswordFile, tokenFile, workerPasswordFile string
|
||||
var noActivate bool
|
||||
fs.StringVar(&opts.Host, "host", "", "SSH server hostname or address")
|
||||
fs.IntVar(&opts.Port, "port", 22, "SSH server port")
|
||||
fs.StringVar(&opts.User, "user", "", "SSH username")
|
||||
fs.StringVar(&opts.IdentityFile, "identity-file", "", "SSH private key path")
|
||||
fs.StringVar(&opts.KeyPassphrase, "key-passphrase", "", "SSH private key passphrase")
|
||||
fs.StringVar(&passphraseFile, "key-passphrase-file", "", "file containing the private key passphrase")
|
||||
fs.StringVar(&opts.Password, "password", "", "SSH login password")
|
||||
fs.StringVar(&passwordFile, "password-file", "", "file containing the SSH login password")
|
||||
fs.StringVar(&opts.SudoPassword, "sudo-password", "", "remote sudo password")
|
||||
fs.StringVar(&sudoPasswordFile, "sudo-password-file", "", "file containing the remote sudo password")
|
||||
fs.StringVar(&opts.KnownHostsFile, "known-hosts", "", "known_hosts path (default: ~/.ssh/known_hosts)")
|
||||
fs.StringVar(&opts.HostKeyFingerprint, "host-key-fingerprint", "", "expected SHA256 SSH host-key fingerprint")
|
||||
fs.BoolVar(&opts.InsecureHostKey, "insecure-host-key", false, "disable SSH host-key verification (unsafe)")
|
||||
fs.StringVar(&opts.Repo, "repo", "", "worker repository to clone/update (default: public rocketgit.ru repo)")
|
||||
fs.StringVar(&opts.Branch, "branch", "", "branch to build; empty resolves the remote default branch")
|
||||
fs.StringVar(&opts.GoVersion, "go-version", "", "Go toolchain version (default: pinned 1.26.0)")
|
||||
fs.StringVar(&opts.GoArch, "go-arch", "", "Go download archive suffix; empty derives it from the remote architecture")
|
||||
fs.StringVar(&opts.BuildDir, "build-dir", "", "remote clone/build directory (default /opt/rsmon-worker-src)")
|
||||
fs.StringVar(&opts.GoModuleProxy, "go-proxy", "", "GOPROXY for the remote build (default: Go default)")
|
||||
fs.StringVar(&opts.ToolchainDir, "toolchain-dir", "", "remote Go install path ending in /go (default /usr/local/go)")
|
||||
fs.StringVar(&opts.StageBinary, "stage-binary", "", "staging binary path (default <build-dir>/rsmon-worker)")
|
||||
fs.DurationVar(&opts.SessionTimeout, "session-timeout", 0, "per-remote-command timeout (default 30m; 0 uses the default)")
|
||||
fs.BoolVar(&noActivate, "no-activate", false, "stop after the staging build and do not install/start a service (activation is the default)")
|
||||
fs.StringVar(&opts.Activation.Name, "name", "", "instance name: activates rsmon-worker-<name> with its own config/data/unit/port")
|
||||
fs.StringVar(&opts.Activation.URL, "url", installer.DefaultURL, "RSMon server URL (RSMON_URL)")
|
||||
fs.StringVar(&opts.Activation.Token, "token", "", "worker API token (RSMON_TOKEN; required for activation)")
|
||||
fs.StringVar(&opts.Activation.Token, "api-key", "", "worker API token (alias for --token)")
|
||||
fs.StringVar(&tokenFile, "token-file", "", "file containing the worker API token")
|
||||
fs.StringVar(&opts.Activation.PublicURL, "public-url", "", "advertised public origin (PUBLIC_URL; scheme + host, no path)")
|
||||
fs.StringVar(&opts.Activation.Host, "worker-host", "", "worker webapp bind address (WORKER_HOST; default 127.0.0.1)")
|
||||
fs.StringVar(&opts.Activation.Port, "worker-port", "", "worker webapp bind port (WORKER_PORT; default 27401, required with --name)")
|
||||
fs.StringVar(&opts.Activation.Login, "worker-login", "", "operator console login (WORKER_LOGIN; default admin with a generated password)")
|
||||
fs.StringVar(&opts.Activation.Password, "worker-password", "", "operator console password (WORKER_PASSWORD)")
|
||||
fs.StringVar(&workerPasswordFile, "worker-password-file", "", "file containing the operator console password")
|
||||
fs.StringVar(&opts.Activation.EnvFile, "env-file", "", "local systemd-safe env file uploaded for activation (overrides the individual knobs)")
|
||||
fs.BoolVar(&opts.Activation.NoStart, "no-start", false, "install the binary, env, data dir, and service definition without starting the worker")
|
||||
fs.Usage = func() {
|
||||
fmt.Fprintln(fs.Output(), "Usage: rsmon-worker source-install --host HOST --user USER [--token TOKEN|--token-file FILE] [SSH options] [source options]")
|
||||
fs.PrintDefaults()
|
||||
}
|
||||
if err := fs.Parse(args); err != nil {
|
||||
if errors.Is(err, flag.ErrHelp) {
|
||||
return 0
|
||||
}
|
||||
return 2
|
||||
}
|
||||
if fs.NArg() != 0 {
|
||||
fs.Usage()
|
||||
return 2
|
||||
}
|
||||
var err error
|
||||
if opts.KeyPassphrase, err = secretValue(opts.KeyPassphrase, passphraseFile); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "source-install failed: %v\n", err)
|
||||
return 1
|
||||
}
|
||||
if opts.Password, err = secretValue(opts.Password, passwordFile); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "source-install failed: %v\n", err)
|
||||
return 1
|
||||
}
|
||||
if opts.SudoPassword, err = secretValue(opts.SudoPassword, sudoPasswordFile); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "source-install failed: %v\n", err)
|
||||
return 1
|
||||
}
|
||||
if opts.Activation.Token, err = secretValue(opts.Activation.Token, tokenFile); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "source-install failed: %v\n", err)
|
||||
return 1
|
||||
}
|
||||
if opts.Activation.Password, err = secretValue(opts.Activation.Password, workerPasswordFile); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "source-install failed: %v\n", err)
|
||||
return 1
|
||||
}
|
||||
opts.Activation.Activate = !noActivate
|
||||
res, err := installer.SourceInstall(opts)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "source-install failed: %v\n", err)
|
||||
return 1
|
||||
}
|
||||
fmt.Printf("source install complete\n")
|
||||
fmt.Printf(" distro: %s\n", res.Detection.Summarize())
|
||||
fmt.Printf(" toolchain: %s (%s) at %s\n", res.Plan.Toolchain.Version, res.GoArch, res.ToolchainDir)
|
||||
fmt.Printf(" branch: %s\n", res.ResolvedBranch)
|
||||
fmt.Printf(" commit: %s\n", res.ResolvedCommit)
|
||||
fmt.Printf(" record file: %s\n", res.RecordFile)
|
||||
fmt.Printf(" staged build: %s\n", res.StageBinary)
|
||||
if res.Activation != nil {
|
||||
fmt.Printf(" installed to: %s\n", res.Activation.Binary)
|
||||
fmt.Printf(" env file: %s (mode 0600)\n", res.Activation.EnvFile)
|
||||
fmt.Printf(" data dir: %s\n", res.Activation.DataDir)
|
||||
if res.Activation.UnitFile != "" {
|
||||
fmt.Printf(" service: %s (%s)\n", res.Activation.UnitFile, res.Activation.UnitName)
|
||||
} else {
|
||||
fmt.Println(" service: none (no supported init system detected; supervisor-managed)")
|
||||
}
|
||||
if res.Activation.Started {
|
||||
fmt.Printf(" status: running (supervisor=%s, /healthz verified)\n", res.Activation.Supervisor)
|
||||
} else {
|
||||
fmt.Printf(" status: installed (not started, --no-start)\n")
|
||||
}
|
||||
} else {
|
||||
fmt.Println(" status: not installed as a service (--no-activate)")
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
@@ -28,7 +28,7 @@ disagree, update both in the same change or mark the discrepancy explicitly.
|
||||
| [network-diagnostics.md](network-diagnostics.md) | Confirmation checks and dedicated diagnostic tasks | Partial |
|
||||
| [critical-check-cluster.md](critical-check-cluster.md) | Raft-backed dispatchless critical checks | Scaffold only |
|
||||
| [public-endpoint-and-identity.md](public-endpoint-and-identity.md) | One HTTPS origin, peer status, CA/mTLS, managed Raft topology | Planned; peer HTTPS partial |
|
||||
| [source-installation.md](source-installation.md) | Go SSH source installer and Docker/OpenSSH test matrix | Planned |
|
||||
| [source-installation.md](source-installation.md) | Go SSH source installer and Docker/OpenSSH test matrix | Partial: harness + planning landed, remote build pending |
|
||||
| [implementation-roadmap.md](implementation-roadmap.md) | Ordered repository work packages and release gates | Active |
|
||||
| [source-plan-migration.md](source-plan-migration.md) | Source-to-target conversion ledger and resolved conflicts | Complete mapping |
|
||||
|
||||
|
||||
@@ -1,7 +1,170 @@
|
||||
# Changelog
|
||||
|
||||
## 2026-08-13
|
||||
|
||||
### Source-install E2E in CI (work package 5)
|
||||
|
||||
- Added `.github/workflows/test-ssh.yml`, a Gitea Actions workflow that runs
|
||||
the Alpine/Ubuntu/Arch Docker/OpenSSH source-install matrix (`make test-ssh`)
|
||||
on pushes to `master` and on manual `workflow_dispatch` only, separate from
|
||||
the Docker-free unit CI. It does not trigger on `pull_request`: the distro
|
||||
fixtures execute the checked-out code inside privileged Docker, so untrusted
|
||||
PR code must never run there automatically. It bounds the job with
|
||||
`timeout-minutes: 90` (the go-test `-timeout 60m` stays in place), scopes
|
||||
concurrency per ref (`test-ssh-${{ gitea.ref }}`), declares
|
||||
`permissions: contents: read`, uploads no artifacts, and cleans up on every
|
||||
path.
|
||||
- Pinned the two actions to immutable full commit SHAs (verified against the
|
||||
GitHub API): `actions/checkout@v4` ->
|
||||
`11d5960a326750d5838078e36cf38b85af677262` and `actions/setup-go@v5` ->
|
||||
`40f1582b2485089dde7abd97c1529aa768e1baff`. The repo-wide convention still
|
||||
leaves `ci.yml`/`docker.yml` on moving tags (accepted, documented risk); see
|
||||
`docs/source-installation.md`.
|
||||
- Wired `RSMON_TEST_IMAGE_ALPINE` / `RSMON_TEST_IMAGE_UBUNTU` /
|
||||
`RSMON_TEST_IMAGE_ARCH` and `RSMON_TEST_DOCKER_DNS` repository variables
|
||||
(all empty by default) so CI can pin per-fixture mirror/snapshot images and
|
||||
a resolver for flaky CI DNS.
|
||||
- Added `scripts/ci/test-ssh.sh`: preflights Docker and the harness's
|
||||
loopback port-publishing requirement with a tiny `docker run -p
|
||||
127.0.0.1::22` probe (fails fast with an actionable message plus
|
||||
diagnostics/fix options instead of a 60m timeout on an unsupported runner),
|
||||
then runs `make test-ssh` and traps `EXIT` to remove every leftover
|
||||
`rsmon-worker-test-*` container/network/image tag. Cleanup filters are
|
||||
anchored to the harness's own prefix/repository so they never touch a shared
|
||||
base image. The workflow adds an `if: always()` cleanup step as a
|
||||
belt-and-suspenders so a killed job never leaves test material on the runner.
|
||||
- External network is fetched live by design (go.dev toolchain, rocketgit.ru
|
||||
source clone, distro repos); operators can pin a resolver via the
|
||||
`RSMON_TEST_DOCKER_DNS` repository variable (comma-separated nameservers,
|
||||
applied as `docker run --dns ...`) and mirror/snapshot overrides via
|
||||
`RSMON_TEST_IMAGE_ALPINE` / `RSMON_TEST_IMAGE_UBUNTU` /
|
||||
`RSMON_TEST_IMAGE_ARCH`.
|
||||
- Fixed a history-dependent test fragility: `TestSourceInstallDirtyCheckoutPreservesStaging`
|
||||
and the rollback test's build-failure step dirty the tracked tree by
|
||||
appending a marker line to `Makefile` instead of `git checkout master~1 --
|
||||
Makefile`, which silently stopped dirtying the tree once the last commit did
|
||||
not touch that file.
|
||||
|
||||
## 2026-08-12
|
||||
|
||||
### Source-install hardening review
|
||||
|
||||
- Fail-closed remote scripts: checkout, branch resolution, and build steps now
|
||||
run under `set -eu` (and package/record steps chain with `&&`), so a failed
|
||||
checkout or fetch can never be masked by a stale `rev-parse` or subsequent
|
||||
command. The checkout step additionally refuses (`git diff --quiet` /
|
||||
`--cached --quiet`) before the destructive `checkout -B`, because `-B`
|
||||
silently discards local changes and would otherwise never fail on a dirty
|
||||
tree. A dirty-tree checkout failure surfaces as a `check out branch` error
|
||||
before the build runs; the new `TestSourceInstallSSHCheckoutFailureNotMasked`
|
||||
unit test and `TestSourceInstallDirtyCheckoutPreservesStaging` Docker test
|
||||
prove the previous staging binary and commit record are preserved
|
||||
byte-for-byte.
|
||||
- Atomic toolchain replacement: the Go toolchain is downloaded, SHA-256
|
||||
verified, extracted into a same-filesystem staging dir, verified to report
|
||||
the target version, and only then swapped into `ToolchainDir` with the prior
|
||||
toolchain moved to a sibling `.go-backup` that is restored on swap failure.
|
||||
A failed download/verify/extract/swap never destroys the prior Go.
|
||||
- Record-after-build pairing: `rsmon-worker.commit` is written only after a
|
||||
successful build, so the record and the staged binary always correspond to
|
||||
the same commit. The build verifies `<stage>.new --version` before an atomic
|
||||
`mv -f` over the previous staging binary; `GOMODCACHE` is now set alongside
|
||||
`GOCACHE` inside the build dir so reruns reuse both caches.
|
||||
- Origin verification: an existing checkout's `remote.origin.url` must exactly
|
||||
match the configured repository before anything is fetched or built.
|
||||
- Repository hardening: only `https://` clone URLs without userinfo are
|
||||
accepted (`ValidateRepoURL`, enforced before dialing and again when
|
||||
planning).
|
||||
- Explicit charset validation for Go version and architecture overrides
|
||||
(`sshinstall.ValidGoVersion` / `ValidGoArch`) before any remote mutation.
|
||||
- Bounded remote execution: each remote command is capped by
|
||||
`--session-timeout` (default 30m) and captured stdout is size-bounded
|
||||
alongside the existing stderr bound; deploy's streaming `runRemote` keeps its
|
||||
historical no-timeout behavior.
|
||||
- The source installer now defaults to the remote's default branch (the public
|
||||
repo publishes `master`) instead of the plan's stale `main` default, while
|
||||
`--branch` still pins an explicit branch that must exist remotely. The
|
||||
README quickstart no longer shows the incorrect `--branch main`.
|
||||
- CLI secret flags keep their compatibility, but docs now explicitly state that
|
||||
file options (`-password-file`, etc.) keep secrets out of argv and shell
|
||||
history while direct flags expose them through the process list.
|
||||
- The harness accepts `RSMON_TEST_DOCKER_DNS` (comma-separated) to pin
|
||||
`docker run --dns` for fixture containers, so internet-facing installs are
|
||||
not at the mercy of a flaky local resolver.
|
||||
|
||||
### Remote source-install execution (work package 3)
|
||||
|
||||
- Added `installer.SourceInstall` (`internal/installer/sourceinstall.go`):
|
||||
executes the source-install flow through the existing SSH transport,
|
||||
reusing the `deploy` command's `SSHOptions` (keys, passphrases,
|
||||
passwords, sudo passwords, known-hosts, pinned fingerprints) and its
|
||||
privilege path. Extracted the shared `SSHOptions` struct and a
|
||||
`sudoWrap` helper so deploy and source install cannot diverge.
|
||||
- Steps implemented: minimal package-prerequisite install per distro
|
||||
(`apk`/`apt`/`pacman`/`dnf`, never a compiler), SHA-256-verified Go 1.26
|
||||
toolchain download/extraction with an idempotent version-skip and temp-dir
|
||||
cleanup, clone-or-update of the public repository (with a bounded 3-attempt
|
||||
retry for transient DNS/TLS/proxy failures), resolution of the remote
|
||||
default branch (a pinned branch must exist remotely), a resolved branch and
|
||||
commit record at `<BuildDir>/rsmon-worker.commit`, and a staging build
|
||||
(`CGO_ENABLED=0`, `-trimpath`, repository `-ldflags`) verified via
|
||||
`--version`. The running service, config, and data directory are untouched
|
||||
(work package 4 boundary).
|
||||
- Security: every interpolated remote value is single-quoted; branch and
|
||||
commit values are strictly validated; no worker token or control-plane
|
||||
credential is sent; sudo passwords travel only over session stdin; remote
|
||||
errors are bounded (stderr truncated in `runRemoteOutput`).
|
||||
- Added unit tests for the remote scripts, option validation, branch/commit
|
||||
parsing, sudo wrapping, the secrets-absent contract, and an in-process
|
||||
real-SSH orchestration flow (with missing-pinned-branch, build-failure, and
|
||||
detection-failure paths).
|
||||
- Added `TestSourceInstallFixtures` to the Docker/OpenSSH harness: each of
|
||||
Alpine, Ubuntu, and Arch installs from a clean state through the real
|
||||
harness transport (prerequisite install, verified Go 1.26, clone, resolved
|
||||
commit, staging build), then a rerun proves idempotency (same branch,
|
||||
toolchain reuse, no temp leaks). All three resolved the public repo's
|
||||
`master` at `4651deb2...` in the recorded run. `make test-ssh` timeout
|
||||
raised to 60m.
|
||||
- Documented the branch-resolution reality: the public repository currently
|
||||
publishes `master`, and the installer records whatever the remote default
|
||||
branch resolves to.
|
||||
|
||||
### Source-install harness and planning foundations (work packages 1-2)
|
||||
|
||||
- Added `internal/installer/harness`: a reusable Docker/OpenSSH test harness
|
||||
that builds real OpenSSH containers for Alpine, Ubuntu, and Arch, waits for
|
||||
real network SSH readiness, captures the server host key into a temp
|
||||
`known_hosts` file, and tears the container, network, per-instance fixture
|
||||
image tag, and temp dir down reliably. It uses the `golang.org/x/crypto/ssh`
|
||||
library and known_hosts verification semantics the installer's `deploy` path
|
||||
relies on (the harness owns its connection code rather than reusing the
|
||||
installer functions) and never mocks SSH. A fresh known_hosts file trusts
|
||||
the first key (TOFU); the host-key mismatch test proves a different key is
|
||||
rejected before any command runs.
|
||||
- Added distro fixtures under `internal/installer/harness/testdata/fixtures`.
|
||||
Alpine defaults to the `reg.rsxx.ru/library/alpine:3` mirror; Ubuntu and Arch
|
||||
fall back to Docker Hub refs overridable via `RSMON_TEST_IMAGE_<NAME>`.
|
||||
Each fixture starts clean (no Go, no worker source) and authenticates with a
|
||||
bundled test key; password auth is disabled. The test key is strictly
|
||||
test-only - it grants root only to the disposable fixture containers - and
|
||||
must never be used outside the harness.
|
||||
- Added opt-in integration controls: the Docker tests run only with
|
||||
`RSMON_TEST_DOCKER=1` (`make test-ssh`); default `make test` and `go test
|
||||
./...` skip them and never pull or start containers. `make test` also pins
|
||||
`RSMON_TEST_DOCKER=0` so an exported opt-in flag cannot leak into the unit
|
||||
run.
|
||||
- Integration tests assert real SSH round trips, clean target state, distro /
|
||||
package-manager / init detection per fixture, a full source plan including
|
||||
the pinned Go toolchain, host-key mismatch rejection, host-key stability,
|
||||
failed-start cleanup, and complete teardown (container, network, fixture
|
||||
image tag, and temp dir gone).
|
||||
- Added `internal/sshinstall`: pure, unit-tested detection and planning for the
|
||||
source installer - os-release parsing, distro/package-manager/init
|
||||
resolution, `uname -m` to Go archive mapping, pinned Go 1.26 toolchain with
|
||||
published SHA-256, and a reviewable ordered plan. No remote execution yet.
|
||||
- `make test` now includes the new packages; `make test-ssh` runs the live
|
||||
fixture matrix.
|
||||
|
||||
### Public endpoint configuration (milestone 1 of public-endpoint-and-identity)
|
||||
|
||||
- `PUBLIC_URL` is now the canonical advertised public origin; the legacy
|
||||
|
||||
@@ -15,10 +15,28 @@ Worker repository:
|
||||
metadata;
|
||||
- add a package/install smoke test for Docker and systemd artifacts;
|
||||
- add the Docker/OpenSSH source-install matrix for Alpine, Ubuntu, and Arch from
|
||||
[source-installation.md](source-installation.md), using Go 1.26 and branch
|
||||
`main`;
|
||||
[source-installation.md](source-installation.md), using Go 1.26 and the
|
||||
remote default branch;
|
||||
- document immutable SHA and release tags as production defaults.
|
||||
|
||||
Source-install foundations landed:
|
||||
|
||||
- [x] reusable Docker/OpenSSH harness and distro fixtures for Alpine, Ubuntu,
|
||||
and Arch (`internal/installer/harness`) with real SSH readiness and
|
||||
reliable teardown, gated behind `RSMON_TEST_DOCKER` (`make test-ssh`);
|
||||
- [x] pure distro/toolchain/source-install planning (`internal/sshinstall`):
|
||||
os-release detection, package-manager/init resolution, pinned Go 1.26
|
||||
toolchain with published SHA-256, and a plan the executor runs;
|
||||
- [x] remote package install, Go download, clone, and build execution through
|
||||
the SSH transport (`installer.SourceInstall`): prerequisite install,
|
||||
SHA-256-verified Go toolchain, clone/update of the public repo, resolved
|
||||
branch/commit record, and a staging build. Running service/config is not
|
||||
touched (source-install work package 3);
|
||||
- [x] atomic service activation, rollback, and failure-preservation tests over
|
||||
SSH (source-install work package 4);
|
||||
- [x] run the full source-install E2E matrix in CI (source-install work
|
||||
package 5).
|
||||
|
||||
Gate: a push publishes `sha-<12>` and `latest` manifests for both platforms,
|
||||
and a container remains healthy when the control plane is unavailable.
|
||||
|
||||
|
||||
@@ -5,8 +5,12 @@ Docker image) into a running, enabled systemd service on a Linux host. It is
|
||||
the supported way to deploy the worker: it writes the configuration, the
|
||||
systemd unit, and the data directory, then starts the service.
|
||||
|
||||
> A Go SSH source installer is planned in
|
||||
> [source-installation.md](source-installation.md). Today
|
||||
> A Go SSH source installer (`rsmon-worker source-install`) builds the worker
|
||||
> from source on a remote host over SSH - prerequisites, verified Go toolchain,
|
||||
> clone/update of the public repository (resolved to the remote default branch
|
||||
> unless pinned), resolved branch/commit record, and a staging build - without
|
||||
> yet installing a service. See
|
||||
> [source-installation.md](source-installation.md). Today the local
|
||||
> `install` copies the binary you invoke it from (or pulls the `--image`
|
||||
> digest), so build first with `make build` and run the resulting
|
||||
> `./bin/rsmon-worker`.
|
||||
|
||||
@@ -88,6 +88,11 @@ signature failure, downgrade, expiry, account change, and unknown critical
|
||||
fields. Credentials remain in memory and are cleared when their signed scope
|
||||
expires.
|
||||
|
||||
Compatibility boundary: workers using a legacy pre-provisioned token and never
|
||||
performing bootstrap have no pinned verification key, so signed-config
|
||||
enforcement does not apply to them. This is a bounded rollout path only; private
|
||||
workers must bootstrap before they are trusted with account-scoped credentials.
|
||||
|
||||
## Public Checks
|
||||
|
||||
Cross-account public-check execution is not the same as normal private scope.
|
||||
@@ -147,11 +152,11 @@ monitor execution.
|
||||
|
||||
## Implementation Work Packages
|
||||
|
||||
1. [ ] Add signed account/config identity to `internal/wire` and runner state.
|
||||
2. [ ] Validate task account and credential scope locally before dispatch.
|
||||
1. [x] Add signed account/config identity to `internal/wire` and runner state.
|
||||
2. [x] Validate task account scope locally before dispatch.
|
||||
3. [x] Implement in-memory reconnect token rotation without stopping worker
|
||||
subsystems.
|
||||
4. [ ] Implement one-time bootstrap, durable token storage, and rotation
|
||||
4. [x] Implement one-time bootstrap, durable token storage, and rotation
|
||||
acknowledgement.
|
||||
5. [ ] Add worker disable/revoke behavior and visible stale-config state.
|
||||
6. [ ] Add mTLS as an optional first transport, then require it for Raft clusters.
|
||||
|
||||
@@ -2,10 +2,41 @@
|
||||
|
||||
## Status
|
||||
|
||||
Planned. The current Go installer can upload a binary or deploy an immutable
|
||||
Docker image over SSH. It does not yet install build prerequisites, download Go,
|
||||
clone the public repository, or build remotely. Existing tests are unit tests;
|
||||
there is no live OpenSSH-container installation test.
|
||||
In progress. Work packages 1-5 are implemented (work package 5 is the
|
||||
Docker/OpenSSH E2E matrix wired into CI):
|
||||
|
||||
- `internal/installer/harness` builds and runs real OpenSSH containers
|
||||
for Alpine, Ubuntu, and Arch, waits for real network readiness, captures
|
||||
the server host key into a temp `known_hosts` file, and tears the
|
||||
container, network, per-instance fixture image tag, and temp dir down
|
||||
reliably. It never mocks SSH and connects with the
|
||||
`golang.org/x/crypto/ssh` library and known_hosts verification
|
||||
semantics the installer's `deploy` path relies on.
|
||||
- `internal/sshinstall` resolves a remote host's distro, package manager,
|
||||
and init system from `/etc/os-release`, plans the pinned Go 1.26
|
||||
toolchain (published SHA-256) for the remote architecture, and produces
|
||||
a pure source-install plan. It executes nothing.
|
||||
- `installer.SourceInstall` (work package 3) executes the plan through
|
||||
the same SSH transport, authentication, and host-key verification the
|
||||
`deploy` command uses. It installs the minimal package prerequisites,
|
||||
downloads and SHA-256-verifies the pinned Go toolchain before
|
||||
extraction, clones/updates the public repository, checks out the
|
||||
resolved branch, records the resolved branch and commit, and builds the
|
||||
worker to a staging path.
|
||||
- Work package 4 (`installer.SourceInstall` with `Activation` enabled)
|
||||
atomically installs the staged binary, validated environment, data
|
||||
directory, and the detected init's service definition; starts/restarts
|
||||
the worker; verifies the process and `/healthz`; and rolls back to the
|
||||
prior working install on any activation/start/health failure. Reruns
|
||||
are idempotent with exactly one running worker and no leaked temp
|
||||
files. The CLI runs the full flow by default and accepts
|
||||
`--no-activate` (staging only) and `--no-start`.
|
||||
|
||||
The current Go installer can also upload a binary or deploy an immutable
|
||||
Docker image over SSH (`deploy`). Source installs build remotely, then
|
||||
activate the staged build atomically with rollback. Existing tests are
|
||||
unit tests plus the harness tests that run against live OpenSSH
|
||||
containers when explicitly enabled.
|
||||
|
||||
## Initial Platform Scope
|
||||
|
||||
@@ -31,16 +62,276 @@ It then:
|
||||
3. downloads the pinned Go 1.26 toolchain for the detected architecture and
|
||||
verifies the published SHA-256;
|
||||
4. clones `https://rocketgit.ru/rsmon/worker.git` or updates an existing clone;
|
||||
5. checks out branch `main` and records the resolved commit;
|
||||
5. checks out the resolved branch (the pinned branch when one is configured,
|
||||
otherwise the remote's default branch) and records the resolved commit;
|
||||
6. builds a reproducible worker binary with the repository build flags;
|
||||
7. atomically installs the binary, validated environment, data directory, and
|
||||
service definition;
|
||||
8. starts the service and verifies process status and `/healthz`.
|
||||
|
||||
Repository, branch, Go version, checksum source, build directory, and Go module
|
||||
proxy may be configurable, but production output records their resolved values.
|
||||
The default repository is publicly readable and requires no source credential.
|
||||
|
||||
## Work Package 3: Remote Execution To A Staging Path
|
||||
|
||||
Work package 3 is `installer.SourceInstall` in
|
||||
`internal/installer/sourceinstall.go`. It reuses the `deploy` command's
|
||||
`SSHOptions` (authentication, sudo password, known-hosts and fingerprint
|
||||
verification) and runs every remote step with the same privilege path
|
||||
(root, passwordless sudo, or `sudo -S -p ''` with the password delivered
|
||||
only over stdin). Steps 1-6 of the flow above are implemented; step 7
|
||||
(atomic install) and step 8 (start + verify) are work package 4.
|
||||
|
||||
Per step:
|
||||
|
||||
- **Prerequisite install.** `packageScript` renders the distro's
|
||||
idempotent command (`apk add --no-cache`, `apt-get update` +
|
||||
`apt-get install -y --no-install-recommends`, `pacman -Sy --noconfirm
|
||||
--needed`, `dnf install -y`) for the minimal plan packages (`git`,
|
||||
`ca-certificates`, `curl`, `tar`, `gzip`). No compiler is ever planned
|
||||
or installed.
|
||||
- **Toolchain.** `toolchainScript` downloads the pinned Go tarball into a
|
||||
`mktemp` temp dir, verifies it with `sha256sum -c -` *before*
|
||||
extraction, extracts into a staging dir on the same filesystem as
|
||||
`ToolchainDir` (which must end in `/go`, default `/usr/local/go`),
|
||||
verifies the staged toolchain reports the target version, and only then
|
||||
swaps it into place. The prior toolchain is moved to a sibling
|
||||
`.go-backup` and is restored if the swap fails, so a failed
|
||||
download/verify/extract/swap always leaves the prior Go untouched. A
|
||||
present toolchain that already reports the target version is reused, so
|
||||
reruns do not re-download. Temp, staging, and backup directories are
|
||||
removed on success and failure.
|
||||
- **Clone/update.** `cloneUpdateScript` clones the repository when
|
||||
`BuildDir` has no `.git` and otherwise fetches with `--prune`, so a
|
||||
rerun updates in place. An existing checkout's `remote.origin.url` must
|
||||
exactly match the configured repository before anything is fetched or
|
||||
built, so the installer can never fetch or build an unconfigured
|
||||
repository. The clone/fetch retries up to three times (2s apart)
|
||||
because real repositories can be transiently unreachable (DNS, TLS, or
|
||||
proxy hiccups); three bounded attempts keep a momentary outage from
|
||||
failing a full source install.
|
||||
- **Resolved branch.** The installer resolves the remote default branch
|
||||
via `git remote set-head origin --auto` +
|
||||
`git symbolic-ref --short refs/remotes/origin/HEAD`. When no branch is
|
||||
pinned it builds the remote default (the public repo currently
|
||||
publishes `master`); a pinned branch must exist remotely or the install
|
||||
fails before the build. The resolved branch and the `git rev-parse
|
||||
HEAD` commit (validated as 40 lowercase hex) are returned by
|
||||
`SourceInstall`.
|
||||
- **Staging build.** `buildScript` builds with `CGO_ENABLED=0`,
|
||||
`-trimpath`, the repository's own `-ldflags` shape (version `dev`,
|
||||
resolved commit short form, UTC build date), and both `GOCACHE` and
|
||||
`GOMODCACHE` inside the build dir (so reruns reuse them), plus an
|
||||
optional `GOPROXY`. The binary is built to a sibling `<stage>.new`,
|
||||
verified with `<stage>.new --version`, and only then atomically swapped
|
||||
over `<BuildDir>/rsmon-worker` (or `StageBinary`), so a failed build
|
||||
never replaces the previous staging binary. It is not written to
|
||||
`/usr/local/bin`.
|
||||
- **Commit record.** The `rsmon-worker.commit` record (a
|
||||
`branch=...` / `commit=...` format in the build dir) is written only
|
||||
*after* a successful build, so the record and the staged binary always
|
||||
correspond to the same commit.
|
||||
|
||||
Security properties of work package 3:
|
||||
|
||||
- Every interpolated value (repository, branch, build dir, URLs, SHA-256,
|
||||
package names, paths) is single-quoted; repository, branch, commit, Go
|
||||
version, and Go architecture values are additionally validated with
|
||||
strict patterns. No worker token or control-plane credential is ever
|
||||
sent: the install stages a binary and touches no service configuration.
|
||||
- The repository must be an `https://` URL without userinfo, so source
|
||||
credentials cannot reach the remote clone command or the clone's
|
||||
config.
|
||||
- Sudo passwords are delivered over the session's stdin only, never in a
|
||||
command string (the same `sudoWrap` path the `deploy` command uses).
|
||||
Direct `--password`/`--sudo-password`/`--key-passphrase` flags remain
|
||||
available but expose the value through the process list and shell
|
||||
history; the CLI docs strongly prefer the `-file` variants. The source
|
||||
installer sends no worker token at all.
|
||||
- Every step script fails closed: `set -eu` (or an explicit retry that
|
||||
exits non-zero) is used, so a failed checkout or fetch can never be
|
||||
masked by a stale subsequent command. The checkout step refuses before
|
||||
the destructive `checkout -B` when the tracked working tree is dirty
|
||||
(`git diff --quiet` / `--cached --quiet`), because `checkout -B` would
|
||||
silently discard local changes; a dirty-tree rerun fails at checkout and
|
||||
leaves the previous staging binary and commit record untouched.
|
||||
- Remote errors are bounded: each step returns a step-labelled error,
|
||||
stderr and captured stdout are size-bounded in `runRemoteOutput`, and
|
||||
each remote command is capped by `--session-timeout` (default 30m).
|
||||
- Toolchain temp, staging, and backup directories are removed on success
|
||||
and failure, and the acceptance test asserts no `/tmp/rsmon-toolchain-*`
|
||||
or `/usr/local/.go-staging-*`/`.go-backup` leaks after the rerun.
|
||||
- Failed builds and failed checkouts leave the previous staging binary
|
||||
untouched (the binary is only overwritten by an atomically-swapped
|
||||
successful build, and a checkout failure aborts before the build).
|
||||
|
||||
## Work Package 4: Atomic Activation And Rollback
|
||||
|
||||
Work package 4 is the second half of `installer.SourceInstall`, driven by
|
||||
`Activation` (`internal/installer/sourceactivate.go`). It runs over the
|
||||
same SSH executor, privilege path, and bounded-error machinery as the
|
||||
staging steps and installs the staged build atomically:
|
||||
|
||||
- **Layout.** The classic installer's on-disk layout is reused
|
||||
(`resolvePaths`): binary `/usr/local/bin/rsmon-worker[-name]`, config
|
||||
`/etc/rsmon-worker[-name]/worker.env` (mode 0600), data
|
||||
`/var/lib/rsmon-worker[-name]`, and a service definition for the
|
||||
detected init (systemd unit `/etc/systemd/system/rsmon-worker.service`
|
||||
mode 0644, or an OpenRC script `/etc/init.d/rsmon-worker` mode 0755).
|
||||
A named instance (`--name`) mirrors the classic multi-instance
|
||||
convention with `-<name>` suffixes and its own unit and port.
|
||||
- **Environment.** The env file is rendered by the classic installer's
|
||||
`resolveInstallEnv` + `renderEnvFile` (so `PUBLIC_URL` canonicalization,
|
||||
basic-auth XOR, required `RSMON_URL`/`RSMON_TOKEN`, and systemd-safe
|
||||
value validation are identical to `install`). The env file is read
|
||||
*exactly once* and validated from the in-memory bytes
|
||||
(`parseEnvironmentContent`), and the rendered env is cached during
|
||||
option normalization and reused at activation time, so a hostile local
|
||||
writer cannot swap the file between the preflight and the remote
|
||||
install (env-file TOCTOU). The rendered bytes are uploaded into a
|
||||
server-created 0700 temp dir (`mktemp -d /tmp/rsmon-worker-act.XXXXXX`
|
||||
under the SSH user) rather than a predictable `/tmp` path, eliminating
|
||||
the symlink/TOCTOU attack surface on the upload while the token still
|
||||
travels only as base64 over the session stdin and, later, only inside
|
||||
the mode-0600 env file. The env is then installed with mode 0600 in the
|
||||
config dir (mode 0750).
|
||||
- **Atomic install.** The staged binary is validated (`<stage>
|
||||
--version`) before any state is touched, then installed with a
|
||||
same-directory temp file + rename. The env and unit files are installed
|
||||
the same way. The data dir (and `webapp/` subdir) is created with mode
|
||||
0755.
|
||||
- **Supervisor.** `restart_svc` drives start/restart through whichever
|
||||
init system is *actually running*: systemd (`systemctl`), OpenRC
|
||||
(`rc-service`), or the embedded supervisor fallback when no init is
|
||||
running (containers/chroots, the "no-service gate"). The health loop
|
||||
verifies the service is active *through the supervisor* — `systemctl
|
||||
is-active` for systemd, `rc-service status` for OpenRC, and a
|
||||
zombie-aware pid check for the embedded supervisor — never through a
|
||||
pid file on the init-managed paths. The embedded supervisor keeps the
|
||||
worker as a single background process with a pid file under the data
|
||||
dir, stops the previous instance before starting, refuses to kill a pid
|
||||
whose executable is not the configured worker (`/proc/<pid>/exe`
|
||||
checked before every `kill`), and answers `/healthz` via the worker's
|
||||
own `liveness` subcommand with a clean `env -i` environment built from
|
||||
the env file (so a rolled-back env can never leak a stale variable into
|
||||
the restored process). The OpenRC init script exports the env-file
|
||||
variables (`set -a` before sourcing) so the worker inherits them when
|
||||
OpenRC starts it.
|
||||
- **Rollback.** The prior binary, env, and unit (and the unit's enable
|
||||
state) are snapshotted into the data dir before any mutation, and an
|
||||
`EXIT`/`HUP`/`INT`/`TERM` trap restores them (preserving the prior
|
||||
binary/env/unit metadata via `cp -p`) and restarts the prior service if
|
||||
any later step fails or a signal interrupts the run. A corrupt staged
|
||||
binary fails the pre-validation *before* the trap is armed, so the
|
||||
prior install is never touched. A fresh-install failure disables the
|
||||
newly-enabled unit (`systemctl disable` / `rc-update del`) and removes
|
||||
it; a rerun failure restores the prior unit and re-applies its prior
|
||||
enable state. Rollback also removes the backup dir, the `.new` temp
|
||||
files, the uploaded env/unit temps, and the activation lock.
|
||||
- **Lock, recovery, interruption.** A `mkdir`-based activation lock
|
||||
(`.rsmon-activate.lock`, broken automatically when its recorded pid is
|
||||
dead) prevents concurrent activations from racing the deployed-state
|
||||
mutation. The snapshot is marked; a run killed mid-flight (SSH drop,
|
||||
SIGKILL) leaves that marker, and the next activation restores the
|
||||
leftover snapshot before proceeding, so the host never stays
|
||||
half-activated.
|
||||
- **Idempotency.** A rerun reuses the existing env (rewriting it
|
||||
deterministically from the same knobs), swaps the binary atomically,
|
||||
restarts exactly one worker, and leaves no `.rsmon-backup`, `.new`,
|
||||
`/tmp/rsmon-worker-act.*`, or lock leftovers. `--no-start` installs the
|
||||
full layout without starting anything and reruns stay idle.
|
||||
|
||||
The `source-install` CLI activates by default; pass `--no-activate` for
|
||||
the staging-only behavior of work package 3. The worker token is required
|
||||
for activation and is supplied with `--token`/`--token-file` or
|
||||
`--env-file` (prefer the file options; direct flags expose the value
|
||||
through the process list).
|
||||
|
||||
## Work Package 5: Alpine/Ubuntu/Arch E2E In CI
|
||||
|
||||
Work package 5 runs the full Docker/OpenSSH E2E matrix on pushes to
|
||||
`master` and on manual `workflow_dispatch` runs. It is a separate Gitea
|
||||
Actions workflow (`.github/workflows/test-ssh.yml`) so the ordinary unit
|
||||
CI run stays Docker-free; the job executes `make test-ssh` through
|
||||
`scripts/ci/test-ssh.sh`.
|
||||
|
||||
CI behavior:
|
||||
|
||||
- **Trust boundary and triggers.** The workflow runs *only* on pushes to
|
||||
`master` (the default branch) and on manual `workflow_dispatch`. It
|
||||
deliberately does **not** trigger on `pull_request`: the distro fixtures
|
||||
execute the checked-out repository code inside privileged Docker
|
||||
containers, so an untrusted PR must never reach the runner's Docker
|
||||
surface automatically. Gitea's read-only token clamp for fork PRs does
|
||||
not limit what containers can do on the runner host, so PR coverage is
|
||||
left to the Docker-free unit CI (`ci.yml`) and to manual dispatch after
|
||||
a human reviews the change.
|
||||
- **Concurrency is scoped per ref.** `concurrency.group:
|
||||
test-ssh-${{ gitea.ref }}` gives master pushes their own group (a newer
|
||||
master push cancels a superseded in-flight master run instead of
|
||||
stacking) and gives a manual dispatch on another branch its own group so
|
||||
it never cancels the master run. Gitea Actions evaluates the expression,
|
||||
and `gitea.ref` is the same documented context the repo's `docker.yml`
|
||||
already uses.
|
||||
- **Least privilege.** The workflow declares `permissions: contents: read`
|
||||
(supported by Gitea Actions as the `GITEA_TOKEN` scope for
|
||||
code/releases), so the job's token can only read the repository; the
|
||||
workflow never writes, pushes, or publishes.
|
||||
- **Action revisions.** The two actions this workflow uses are pinned to
|
||||
immutable full commit SHAs (not moving tags): `actions/checkout@v4` ->
|
||||
`11d5960a326750d5838078e36cf38b85af677262` and `actions/setup-go@v5` ->
|
||||
`40f1582b2485089dde7abd97c1529aa768e1baff` (verified 2026-08-13 against
|
||||
the GitHub API that each tag points to a commit object). The repo-wide
|
||||
convention still leaves `ci.yml` and `docker.yml` on moving tags
|
||||
(`@v4`, `@v5`, `@v3`, `@v6`); that is an accepted, documented risk: a
|
||||
tag move can change behavior without a workflow diff. New workflows
|
||||
should pin SHAs like this one; migrating the existing workflows is a
|
||||
separate change.
|
||||
- **Runner requirement.** The harness dials fixture SSH ports published on
|
||||
the Docker daemon's `127.0.0.1`, so the job must share the daemon's
|
||||
loopback (a host-mode runner or a job container with host networking).
|
||||
`scripts/ci/test-ssh.sh` probes this with a tiny `docker run -p
|
||||
127.0.0.1::22` round trip *before* the matrix and fails fast with a
|
||||
clear, actionable message (including diagnostics and fix options)
|
||||
instead of after a 60m go-test timeout.
|
||||
- **Bounding and timeout.** The workflow sets `timeout-minutes: 90` and
|
||||
the existing `make test-ssh` go-test `-timeout 60m` stays in place, so
|
||||
the whole job is hard-bounded even under slow networks or downloads.
|
||||
- **Cleanup.** The harness already tears down every container, network,
|
||||
per-instance fixture image tag, and temp dir on success and failure.
|
||||
`scripts/ci/test-ssh.sh` additionally traps `EXIT` to remove any
|
||||
leftover `rsmon-worker-test-*` resource (container, network, or the
|
||||
per-instance `rsmon-worker-test/<fixture>-<suffix>:local` image tag), and
|
||||
the workflow adds an `if: always()` step that does the same even when the
|
||||
script itself is killed. All filters are anchored to the harness's own
|
||||
prefix and image repository (`name=^rsmon-worker-test-`,
|
||||
`reference=rsmon-worker-test/*`), so cleanup never touches a shared base
|
||||
image (`reg.rsxx.ru/library/alpine:3`, `ubuntu:24.04`, `archlinux:latest`)
|
||||
or an unrelated resource.
|
||||
- **Mirrors and overrides.** The Alpine fixture already uses the
|
||||
`reg.rsxx.ru/library/alpine:3` mirror. Ubuntu and Arch have no mirror yet
|
||||
and default to Docker Hub. The workflow wires the
|
||||
`RSMON_TEST_IMAGE_ALPINE`, `RSMON_TEST_IMAGE_UBUNTU`, and
|
||||
`RSMON_TEST_IMAGE_ARCH` repository variables (empty by default) so an
|
||||
operator can pin a mirror or a specific distro snapshot per fixture, and
|
||||
the `RSMON_TEST_DOCKER_DNS` repository variable (comma-separated
|
||||
nameservers, applied as `docker run --dns ...`) to pin a resolver for
|
||||
flaky CI DNS. External network (go.dev toolchain download, the public
|
||||
rocketgit.ru source clone, distro package repos) is fetched live by
|
||||
design.
|
||||
- **Artifacts and secrets.** The workflow uploads no artifacts: the go-test
|
||||
log stays in the runner's job log and nothing private (test key,
|
||||
`known_hosts`, env files) is retained on the runner or published. The
|
||||
fixture containers receive only the fixed `e2e-activation-test-token`
|
||||
and the bundled test-only key, never a real worker token.
|
||||
- **Caching.** `actions/setup-go` caches the Go module/build cache used to
|
||||
compile the harness test binary (public dependencies only). Fixture and
|
||||
toolchain downloads are not cached because they run inside disposable
|
||||
distro containers; reruns rebuild them cleanly.
|
||||
|
||||
Work package 5 does not change how the source installer behaves. It only
|
||||
adds a CI surface for the existing acceptance tests.
|
||||
|
||||
## Docker OpenSSH Test Harness
|
||||
|
||||
Adapt the real-network pattern from `/data/_swap/sshkeymanager`: start an
|
||||
@@ -58,32 +349,247 @@ Each clean target begins without Go or the worker source. The test asserts
|
||||
package installation, verified Go version, clone branch/resolved commit, build,
|
||||
atomic config permissions, running service where supported, and HTTP liveness.
|
||||
|
||||
### Implementation (`internal/installer/harness`)
|
||||
|
||||
The harness starts one container per distro fixture
|
||||
(`testdata/fixtures/{alpine,ubuntu,arch}/Dockerfile`), waits for a real
|
||||
TCP + SSH handshake, captures the server host key into a temp
|
||||
`known_hosts` file, and connects with the `golang.org/x/crypto/ssh`
|
||||
library and known_hosts verification semantics the installer's `deploy`
|
||||
uses. The harness owns its connection code rather than calling into the
|
||||
installer package, so the tests stay independent; only the library and
|
||||
the verification semantics are shared.
|
||||
|
||||
Host-key handling is trust-on-first-use (TOFU): the fresh temp
|
||||
`known_hosts` file accepts the first key the server presents. What the
|
||||
harness proves is that a known_hosts entry carrying a *different* key is
|
||||
rejected before any command runs (the `TestHarnessHostKeyMismatch` test),
|
||||
not that a fingerprint is pinned.
|
||||
|
||||
The fixtures install over the public internet, so environments with flaky
|
||||
local resolvers can pin a reliable upstream via the comma-separated
|
||||
`RSMON_TEST_DOCKER_DNS` variable (applied as `docker run --dns ...`); it
|
||||
is empty by default, keeping Docker's embedded DNS.
|
||||
|
||||
Teardown (`docker rm -f` + `docker network rm` + per-instance
|
||||
`docker image rm` + temp-dir removal) is idempotent, runs on every
|
||||
`Start` error path, and is verified by a dedicated test. Each harness
|
||||
instance builds its own uniquely-tagged fixture image
|
||||
(`rsmon-worker-test/<fixture>-<suffix>:local`), so removing one never
|
||||
deletes a shared base image or another instance's image.
|
||||
|
||||
Base images, mirror-first:
|
||||
|
||||
| Fixture | Default image | Note |
|
||||
| --- | --- | --- |
|
||||
| alpine | `reg.rsxx.ru/library/alpine:3` | reg.rsxx.ru mirror exists |
|
||||
| ubuntu | `ubuntu:24.04` | no mirror yet; override `RSMON_TEST_IMAGE_UBUNTU` |
|
||||
| arch | `archlinux:latest` | no mirror yet; override `RSMON_TEST_IMAGE_ARCH` |
|
||||
|
||||
Any `RSMON_TEST_IMAGE_<NAME>` environment variable overrides the fixture
|
||||
image, so a mirror or local cache can be used when available.
|
||||
|
||||
> **Mutable test images.** The fixture base tags above are deliberately
|
||||
> mutable (a major-tag mirror ref and Docker Hub rolling tags) so the
|
||||
> fixtures track current distro releases. Fixture builds are therefore
|
||||
> not byte-reproducible; the worker's *source-install* production output
|
||||
> pins immutable artifacts (a Go toolchain SHA-256, a branch's resolved
|
||||
> commit) and this harness's image override is the escape hatch for
|
||||
> reproducing a specific distro snapshot.
|
||||
|
||||
The fixtures authenticate with the bundled test key
|
||||
(`testdata/keys/rsmon_test_ed25519`); password auth is disabled and
|
||||
`PermitRootLogin` is `prohibit-password`. `openrc` is installed in the
|
||||
Alpine fixture so init detection has a stable marker; Ubuntu and Arch
|
||||
carry systemd markers.
|
||||
|
||||
> **Test-only key.** The bundled keypair is strictly a test fixture: it
|
||||
> grants root SSH access only to the disposable containers that bake its
|
||||
> public key. It must never be used for real hosts, added to production
|
||||
> images, or treated as a credential outside the harness.
|
||||
|
||||
### Opt-in integration test controls
|
||||
|
||||
Ordinary unit runs never pull or start Docker. The Docker/OpenSSH tests
|
||||
are gated behind the `RSMON_TEST_DOCKER` environment variable:
|
||||
|
||||
- `make test` (default CI unit run) pins `RSMON_TEST_DOCKER=0` and skips
|
||||
every harness Docker test, even when the flag is exported in the
|
||||
developer's environment.
|
||||
- `make test-ssh` sets `RSMON_TEST_DOCKER=1` and runs the full fixture
|
||||
matrix (`TestHarnessFixtures` for alpine/ubuntu/arch, host-key
|
||||
mismatch, stable host key, failed-start cleanup, and teardown
|
||||
assertions).
|
||||
- The harness `Start` itself refuses to run without the opt-in flag.
|
||||
|
||||
Run them locally with:
|
||||
|
||||
```bash
|
||||
make test-ssh
|
||||
```
|
||||
|
||||
or, equivalently:
|
||||
|
||||
```bash
|
||||
RSMON_TEST_DOCKER=1 go test -v -count=1 -timeout 30m ./internal/installer/harness
|
||||
```
|
||||
|
||||
## Idempotency And Security
|
||||
|
||||
- A second run updates/fetches safely and leaves one active service.
|
||||
- Wrong host fingerprints fail before remote mutation.
|
||||
- Tokens/passwords come from files or stdin-safe channels and never appear in
|
||||
command arguments, logs, source checkout, or shell history.
|
||||
- Remote temporary files are removed on success and failure.
|
||||
- Failed builds do not replace a working binary or service definition.
|
||||
- A second run updates/fetches safely: the toolchain is reused when the version
|
||||
matches, the clone's origin is verified against the configured repository and
|
||||
then fetched in place, the resolved branch/commit record is rewritten after
|
||||
the new build succeeds, and the staging build atomically swaps over the
|
||||
previous staging binary. Activation (work package 4) makes the rerun
|
||||
idempotent at the service level too: exactly one worker process exists, the
|
||||
env file is rewritten deterministically from the same knobs, and no
|
||||
`.rsmon-backup`, `.new`, `/tmp/rsmon-worker-act.*`, or lock files leak.
|
||||
- Wrong host fingerprints fail before remote mutation. The harness's fresh
|
||||
known_hosts file is trust-on-first-use; its dedicated mismatch test dials
|
||||
against a known_hosts entry carrying a different server key and proves the
|
||||
dial fails before any command runs.
|
||||
- Tokens/passwords should come from files: `--key-passphrase-file`,
|
||||
`--password-file`, `--sudo-password-file`, and the activation
|
||||
`--token-file`/`--env-file` keep secrets out of argv and shell history,
|
||||
while the equivalent direct flags expose them through the process list. The
|
||||
worker token is written only to the mode-0600 env file, never echoed into a
|
||||
command or log, and sudo passwords travel only over the session's stdin. The
|
||||
env and unit uploads land in a server-created 0700 `mktemp -d` directory (not
|
||||
a predictable `/tmp` path), so no local user can plant a symlink at the
|
||||
upload target (TOCTOU), and the env file itself is read/rendered exactly once
|
||||
so a local writer cannot swap it between validation and upload.
|
||||
- Remote temporary files are removed on success and failure, including the
|
||||
activation backup dir, the uploaded env/unit temps, and the activation lock.
|
||||
The snapshot is written under a marker so a run killed mid-flight is
|
||||
recovered (restored) by the next activation instead of leaving the host
|
||||
half-activated.
|
||||
- Failed builds, failed checkouts, and failed toolchain swaps do not replace a
|
||||
working binary or service definition. Failed activations, failed starts, and
|
||||
failed `/healthz` verifications restore the prior binary, env, and service
|
||||
definition (preserving their metadata), re-apply the prior unit enable state
|
||||
(or disable a freshly-enabled unit on a fresh failure), and bring the prior
|
||||
worker back up (work package 4 rollback). The supervisor only ever kills a
|
||||
process whose `/proc/<pid>/exe` matches the configured worker binary, so a
|
||||
stale or recycled pid file can never kill an unrelated process.
|
||||
- Package-manager and download failures return bounded actionable errors.
|
||||
- The installer verifies Go tarball checksum before extraction.
|
||||
|
||||
## Implementation Work Packages
|
||||
|
||||
1. Add reusable Docker/OpenSSH harness and distro fixtures.
|
||||
2. Add pure distro/toolchain/source-install script planning and unit tests.
|
||||
3. Execute source installation through the existing SSH transport.
|
||||
4. Add atomic build/install, idempotency, and failure rollback.
|
||||
5. Add Alpine, Ubuntu, and Arch network E2E tests to CI.
|
||||
6. Add CentOS-family support.
|
||||
7. Plan native Windows service and macOS launchd installers separately.
|
||||
- [x] 1. Add reusable Docker/OpenSSH harness and distro fixtures.
|
||||
- [x] 2. Add pure distro/toolchain/source-install script planning and unit tests
|
||||
(detection + planning foundation; remote execution is work package 3).
|
||||
- [x] 3. Execute source installation through the existing SSH transport
|
||||
(prerequisites, verified Go toolchain, clone/update, resolved branch and
|
||||
commit record, and a build to a staging path; no service activation).
|
||||
- [x] 4. Add atomic build/install, idempotency, and failure rollback (binary,
|
||||
env, data dir, and init service definition installed atomically; process
|
||||
and `/healthz` verified; activation/start/health failures restore the
|
||||
prior install; reruns keep exactly one service).
|
||||
- [x] 5. Add Alpine, Ubuntu, and Arch network E2E tests to CI.
|
||||
- [ ] 6. Add CentOS-family support.
|
||||
- [ ] 7. Plan native Windows service and macOS launchd installers separately.
|
||||
|
||||
## Acceptance Gates
|
||||
|
||||
- All three initial Linux images install from a clean state through OpenSSH.
|
||||
- The built worker reports the expected version/commit and serves `/healthz`.
|
||||
- Re-running the installer succeeds without duplicate services or leaked files.
|
||||
- Host-key, checksum, clone, build, and service-start failure tests preserve the
|
||||
previous installation.
|
||||
- CI uses approved registry mirrors and cleans every test container/network.
|
||||
- [x] All three initial Linux images install from a clean state through OpenSSH:
|
||||
package install, verified Go 1.26, clone/update, resolved commit, staging
|
||||
build, and (work package 4) atomic activation with a running verified
|
||||
worker.
|
||||
- [x] The built worker reports the expected version/commit and serves `/healthz`
|
||||
(verified by the E2E fixture tests and by the activation health gate).
|
||||
- [x] Re-running the installer succeeds without duplicate services or leaked
|
||||
files (exactly one worker process and no backup/temp leftovers asserted by
|
||||
the rerun test on all three fixtures).
|
||||
- [x] Host-key, checksum, clone, build, and service-start failure tests preserve
|
||||
the previous installation (build/checkout, activation, start, and health
|
||||
failure reruns on the Alpine fixture all restore the prior binary, env,
|
||||
and running service).
|
||||
- [x] CI uses approved registry mirrors and cleans every test container/network.
|
||||
|
||||
## Verified Test Evidence (work packages 1-4)
|
||||
|
||||
Recorded 2026-08-12 from `make test-ssh` (Docker Engine 29.7.1):
|
||||
|
||||
- Alpine `3.24.1` (mirror `reg.rsxx.ru/library/alpine:3`): distro alpine,
|
||||
pkg apk, init openrc.
|
||||
- Ubuntu `24.04` (`VERSION_ID=24.04`): distro ubuntu, pkg apt, init systemd.
|
||||
- Arch rolling image `archlinux:latest` (`VERSION_ID=20260809.0.570793`):
|
||||
distro arch, pkg pacman, init systemd.
|
||||
- Host-key mismatch, host-key stability, failed-start cleanup, and complete
|
||||
teardown (container, network, fixture image tag, and temp dir gone) tests
|
||||
pass; no test container, network, or image tag is left behind.
|
||||
- `TestSourceInstallFixtures` runs the full work-package-3 flow on each
|
||||
fixture from a clean state over the harness's real OpenSSH transport:
|
||||
prerequisite install, verified Go 1.26 toolchain download/extraction, clone
|
||||
of the public repository, resolution of its default branch, resolved-commit
|
||||
record file, and a staging build that reports the resolved commit via
|
||||
`--version`. A rerun succeeds, keeps the same branch, reuses the toolchain,
|
||||
and leaves no `/tmp/rsmon-toolchain-*` temp dirs.
|
||||
- `TestSourceInstallActivationFixtures` runs the full work-package-4 flow on
|
||||
each fixture from a clean state: after the staging build the installer
|
||||
atomically installs the binary (`/usr/local/bin/rsmon-worker`), env
|
||||
(`/etc/rsmon-worker/worker.env`, mode 0600; config dir 0750), data dir
|
||||
(`/var/lib/rsmon-worker`), and the detected init's service definition
|
||||
(systemd unit on Ubuntu/Arch, OpenRC script on Alpine), then starts the
|
||||
worker via the embedded supervisor (no init runs inside the fixtures) and
|
||||
verifies exactly one worker process and a live `/healthz`. A rerun succeeds,
|
||||
keeps a single process, rewrites the env deterministically, and leaves no
|
||||
`.rsmon-backup`, `.new`, `/tmp/rsmon-worker-act.*`, or lock leftovers.
|
||||
- `TestSourceInstallActivationFailureRollback` (Alpine) forces four rerun
|
||||
failures and proves each restores the prior install byte-for-byte and
|
||||
running: a dirty checkout (build failure), a sabotaged atomic swap
|
||||
(activation failure), `WORKER_CLUSTER_ENABLED=true` without credentials (the
|
||||
new process exits at boot; start failure), and `WORKER_HOST=255.255.255.255`
|
||||
(the new process runs but `/healthz` is unreachable; health failure). In
|
||||
every case the installed binary SHA-256, env, service definition, single
|
||||
process, and liveness match the pre-failure state, and no backup, lock, or
|
||||
temp files leak.
|
||||
- `TestSourceInstallActivationNoStart` installs the full layout without
|
||||
starting anything, and reruns stay idle.
|
||||
- The targeted shell fixture tests (`TestActivateShell*`) execute the real
|
||||
activation script against stub `systemctl`/`rc-service`/`rc-update`
|
||||
implementations and cover the init-managed paths the Docker fixtures cannot
|
||||
reach: systemd unit install/enable with a `systemctl is-active`-based health
|
||||
loop (and no pid file), OpenRC install/enable with an `rc-service`-based
|
||||
health loop, fresh-failure rollback disabling a newly-enabled unit, rerun
|
||||
rollback restoring the prior unit and its enable state, stale-lock breaking,
|
||||
and interrupted-run recovery from a leftover backup marker.
|
||||
- The unit suite covers the per-step remote scripts, the activation script
|
||||
(backup + recovery marker, activation lock, atomic install, rollback trap,
|
||||
supervisor, pid/zombie handling, pid-belongs-to-binary checks, enable-state
|
||||
restoration, secrets absent), OpenRC unit rendering (including env
|
||||
export), option validation, branch/commit parsing, sudo wrapping (password
|
||||
never in the command), and an in-process real-SSH orchestration flow with
|
||||
failure paths for missing pinned branches, build failures, detection
|
||||
failures, and activation/start/health failures.
|
||||
- **Residual limitation (stated honestly):** live systemd and OpenRC cannot
|
||||
run inside the Docker/OpenSSH harness (the container PID 1 is sshd), so the
|
||||
init-managed supervisor paths are verified by the stub-based shell fixture
|
||||
tests above rather than against a real init. The stub tools simulate unit
|
||||
state and command flow, not real systemd/OpenRC unit semantics; a real
|
||||
init-system smoke test on a booted host remains a follow-up.
|
||||
|
||||
Work package 5 (CI) evidence, recorded 2026-08-13:
|
||||
|
||||
- `.github/workflows/test-ssh.yml` runs `scripts/ci/test-ssh.sh` on pushes
|
||||
to `master` and on manual `workflow_dispatch` only (never on
|
||||
`pull_request`, because the fixtures execute the checked-out code inside
|
||||
privileged Docker). Concurrency is scoped per ref
|
||||
(`test-ssh-${{ gitea.ref }}`), the job token is `contents: read`, and
|
||||
the script's loopback port-publishing probe passes on a host Docker
|
||||
daemon while its `EXIT` cleanup leaves zero leftover containers,
|
||||
networks, or fixture image tags.
|
||||
- A full local `make test-ssh` run (the exact command the CI job executes)
|
||||
passes the complete harness suite: `TestHarnessFixtures` (alpine/ubuntu/
|
||||
arch), host-key mismatch and stability, teardown, `TestSourceInstallFixtures`
|
||||
(work package 3 staging on all three distros), `TestSourceInstallActivationFixtures`
|
||||
(work package 4 activation on all three), activation failure rollback,
|
||||
`--no-start`, and the dirty-checkout preservation test. Pinned
|
||||
`RSMON_TEST_DOCKER_DNS=1.1.1.1` was used because the local network's
|
||||
default resolvers intermittently time out on `rocketgit.ru`.
|
||||
- `TestSourceInstallDirtyCheckoutPreservesStaging` and the rollback test's
|
||||
build-failure step now dirty the tracked tree deterministically (appending
|
||||
a marker line to `Makefile`) instead of `git checkout master~1 -- Makefile`,
|
||||
which depended on the last two commits differing and silently stopped
|
||||
dirtying the tree once a commit did not touch that file.
|
||||
|
||||
@@ -122,6 +122,25 @@ func (c *Client) RotateToken(ctx context.Context) (string, error) {
|
||||
return out.AuthToken, nil
|
||||
}
|
||||
|
||||
func (c *Client) Bootstrap(ctx context.Context, workerID, token string) (wire.BootstrapResponse, error) {
|
||||
var out wire.BootstrapResponse
|
||||
resp, err := c.postJSONContext(ctx, "/api/internal/workers/bootstrap", wire.BootstrapRequest{WorkerID: workerID, BootstrapToken: token})
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
defer resp.Body.Close() //nolint:errcheck
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return out, checkStatusCode(resp, "bootstrap")
|
||||
}
|
||||
if err := json.NewDecoder(resp.Body).Decode(&out); err != nil {
|
||||
return out, err
|
||||
}
|
||||
if out.AuthToken == "" || out.WorkerID != workerID || out.ConfigVerificationKey == "" {
|
||||
return out, fmt.Errorf("invalid bootstrap response")
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// GetJobs fetches available check jobs from the control plane
|
||||
func (c *Client) GetJobs() (*wire.JobsResponse, error) {
|
||||
httpReq, err := http.NewRequest("GET", c.endpoint+"/api/internal/workers/jobs", http.NoBody)
|
||||
|
||||
@@ -111,8 +111,11 @@ func (c HTTPConfig) IsListenConfigured() bool {
|
||||
// Config holds the local worker connection settings.
|
||||
// Runtime settings are delivered by the control plane over websocket.
|
||||
type Config struct {
|
||||
URL string
|
||||
Token string
|
||||
URL string
|
||||
Token string
|
||||
BootstrapToken string
|
||||
StateFile string
|
||||
WorkerID string
|
||||
|
||||
// MaxConcurrency caps the worker pool size and the number of dispatcher
|
||||
// goroutines. A value <= 0 falls back to DefaultMaxConcurrency.
|
||||
@@ -127,9 +130,12 @@ type Config struct {
|
||||
// ConfigFromEnv creates a Config from environment variables.
|
||||
func ConfigFromEnv() Config {
|
||||
return Config{
|
||||
URL: normalizeURL(os.Getenv("RSMON_URL")),
|
||||
Token: os.Getenv("RSMON_TOKEN"),
|
||||
HTTP: HTTPConfigFromEnv(),
|
||||
URL: normalizeURL(os.Getenv("RSMON_URL")),
|
||||
Token: os.Getenv("RSMON_TOKEN"),
|
||||
BootstrapToken: os.Getenv("RSMON_BOOTSTRAP_TOKEN"),
|
||||
StateFile: strings.TrimSpace(os.Getenv("RSMON_STATE_FILE")),
|
||||
WorkerID: strings.TrimSpace(os.Getenv("RSMON_WORKER_ID")),
|
||||
HTTP: HTTPConfigFromEnv(),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -2,10 +2,13 @@ package distworker
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/ed25519"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
@@ -27,6 +30,8 @@ const (
|
||||
minQueueCapacity = 16
|
||||
|
||||
malformedTaskEnvelopeError = "malformed_task_envelope"
|
||||
finalDrainLimit = 32
|
||||
finalDrainTimeout = 5 * time.Second
|
||||
)
|
||||
|
||||
// jobPool is the minimal interface the Runner needs from a worker pool. It
|
||||
@@ -176,6 +181,8 @@ type Runner struct {
|
||||
workerVersion string
|
||||
capsMu sync.RWMutex
|
||||
workerCaps []string
|
||||
identityMu sync.RWMutex
|
||||
state workerState
|
||||
serverID atomic.Int64
|
||||
metricGeneration atomic.Uint64
|
||||
nextMetricGeneration atomic.Uint64
|
||||
@@ -188,6 +195,8 @@ type Runner struct {
|
||||
controlWriteMu *sync.Mutex
|
||||
reconnectCh chan struct{}
|
||||
rotationMu sync.Mutex
|
||||
leasesMu sync.Mutex
|
||||
leases map[string]string
|
||||
|
||||
// outbox holds messages removed from a per-connection writer queue that
|
||||
// could not be written before its websocket closed.
|
||||
@@ -231,6 +240,7 @@ func NewRunner(cfg *Config) *Runner {
|
||||
resultsBuf: newResultBuffer(),
|
||||
notificationsBuf: newNotificationBuffer(),
|
||||
peerCache: newPeerCache(),
|
||||
leases: make(map[string]string),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -239,7 +249,7 @@ func (r *Runner) Start() error {
|
||||
log.Println("worker: starting...")
|
||||
r.lifecycleMu.Lock()
|
||||
|
||||
if r.config.URL == "" || r.config.Token == "" {
|
||||
if r.config.URL == "" || (r.config.Token == "" && r.config.BootstrapToken == "") {
|
||||
r.lifecycleMu.Unlock()
|
||||
return fmt.Errorf("RSMON_URL and RSMON_TOKEN must be set")
|
||||
}
|
||||
@@ -253,6 +263,34 @@ func (r *Runner) Start() error {
|
||||
return fmt.Errorf("worker: runner stopped")
|
||||
}
|
||||
|
||||
if r.config.BootstrapToken != "" && (r.config.StateFile == "" || !filepath.IsAbs(r.config.StateFile)) {
|
||||
r.lifecycleMu.Unlock()
|
||||
return fmt.Errorf("bootstrap requires an absolute RSMON_STATE_FILE")
|
||||
}
|
||||
state, err := loadWorkerState(r.config.StateFile)
|
||||
if err != nil {
|
||||
r.lifecycleMu.Unlock()
|
||||
return fmt.Errorf("worker state: %w", err)
|
||||
}
|
||||
if state.Token != "" {
|
||||
r.config.Token = state.Token
|
||||
}
|
||||
if r.config.Token == "" {
|
||||
client := NewClient(r.config.URL, "")
|
||||
boot, err := client.Bootstrap(context.Background(), r.config.WorkerID, r.config.BootstrapToken)
|
||||
if err != nil {
|
||||
r.lifecycleMu.Unlock()
|
||||
return fmt.Errorf("worker bootstrap: %w", err)
|
||||
}
|
||||
state = workerState{Token: boot.AuthToken, WorkerID: boot.WorkerID, VerificationKey: boot.ConfigVerificationKey, SigningKeyID: boot.SigningKeyID}
|
||||
if err := saveWorkerState(r.config.StateFile, state); err != nil {
|
||||
r.lifecycleMu.Unlock()
|
||||
return fmt.Errorf("worker state: %w", err)
|
||||
}
|
||||
r.config.Token = boot.AuthToken
|
||||
r.config.BootstrapToken = ""
|
||||
}
|
||||
r.state = state
|
||||
r.clientMu.Lock()
|
||||
r.client = NewClient(r.config.URL, r.config.Token)
|
||||
r.clientMu.Unlock()
|
||||
@@ -344,6 +382,7 @@ func (r *Runner) Start() error {
|
||||
// Stop gracefully stops the worker
|
||||
func (r *Runner) Stop() {
|
||||
r.lifecycleMu.Lock()
|
||||
r.drainFinalResults()
|
||||
select {
|
||||
case <-r.stopCh:
|
||||
// already closed
|
||||
@@ -681,7 +720,21 @@ func (r *Runner) runWebsocket() error {
|
||||
return err
|
||||
}
|
||||
if (msg.Kind == "init" || msg.Kind == "config") && msg.Init != nil {
|
||||
r.applyInit(msg.Init)
|
||||
if err := r.applyInit(msg.Init); err != nil {
|
||||
log.Printf("worker: rejected config: %v", err)
|
||||
continue
|
||||
}
|
||||
if msg.Init.RotateToken != "" && msg.Init.RotationID != "" {
|
||||
r.applyRotation(conn, &writeMu, msg.Init.RotateToken, msg.Init.RotationID)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if msg.Kind == "stale_lease_ack" && msg.StaleLeaseAck != nil {
|
||||
r.leasesMu.Lock()
|
||||
if r.leases[msg.StaleLeaseAck.JobID] == msg.StaleLeaseAck.LeaseToken {
|
||||
delete(r.leases, msg.StaleLeaseAck.JobID)
|
||||
}
|
||||
r.leasesMu.Unlock()
|
||||
continue
|
||||
}
|
||||
if msg.Kind != "task" {
|
||||
@@ -709,6 +762,9 @@ func (r *Runner) enqueueTaskMessage(msg wire.WorkerMessage) bool { //nolint:gocr
|
||||
if msg.NotificationTask.JobID == "" || msg.NotificationTask.LeaseToken == "" {
|
||||
return true
|
||||
}
|
||||
if !r.taskInScope(msg.NotificationTask.AccountID) {
|
||||
return r.enqueueFailedNotification(*msg.NotificationTask, "account_scope_mismatch")
|
||||
}
|
||||
log.Printf("worker: received websocket notification task %s method=%s", msg.NotificationTask.JobID, msg.NotificationTask.Method)
|
||||
return r.EnqueueNotification(*msg.NotificationTask)
|
||||
case msg.Task != nil:
|
||||
@@ -716,6 +772,9 @@ func (r *Runner) enqueueTaskMessage(msg wire.WorkerMessage) bool { //nolint:gocr
|
||||
return true
|
||||
}
|
||||
log.Printf("worker: received websocket task %s", msg.Task.JobID)
|
||||
if !r.taskInScope(msg.Task.AccountID) {
|
||||
return r.enqueueFailedCheck(*msg.Task, "account_scope_mismatch")
|
||||
}
|
||||
if !checkexec.SupportsKind(msg.Task.Kind) {
|
||||
return r.enqueueUnsupportedCheck(*msg.Task)
|
||||
}
|
||||
@@ -733,12 +792,16 @@ func (r *Runner) enqueueTaskEnvelope(envelope *wire.TaskEnvelope) bool {
|
||||
if !matchingEnvelopeJobID(envelope.JobID, envelope.Job.JobID) || envelope.Job.LeaseToken == "" {
|
||||
return true
|
||||
}
|
||||
if !r.taskInScope(envelope.Job.AccountID) {
|
||||
return r.enqueueFailedCheck(*envelope.Job, "account_scope_mismatch")
|
||||
}
|
||||
if envelope.Type != wire.TaskTypeCheck {
|
||||
return r.enqueueFailedCheck(*envelope.Job, malformedTaskEnvelopeError)
|
||||
}
|
||||
if !checkexec.SupportsKind(envelope.Job.Kind) {
|
||||
return r.enqueueUnsupportedCheck(*envelope.Job)
|
||||
}
|
||||
r.rememberLease(envelope.Job.JobID, envelope.Job.LeaseToken)
|
||||
return r.Enqueue(*envelope.Job)
|
||||
}
|
||||
if !matchingEnvelopeJobID(envelope.JobID, envelope.Notify.JobID) || envelope.Notify.LeaseToken == "" {
|
||||
@@ -747,9 +810,97 @@ func (r *Runner) enqueueTaskEnvelope(envelope *wire.TaskEnvelope) bool {
|
||||
if envelope.Type != wire.TaskTypeNotification {
|
||||
return r.enqueueFailedNotification(*envelope.Notify, malformedTaskEnvelopeError)
|
||||
}
|
||||
if !r.taskInScope(envelope.Notify.AccountID) {
|
||||
return r.enqueueFailedNotification(*envelope.Notify, "account_scope_mismatch")
|
||||
}
|
||||
r.rememberLease(envelope.Notify.JobID, envelope.Notify.LeaseToken)
|
||||
return r.EnqueueNotification(*envelope.Notify)
|
||||
}
|
||||
|
||||
func (r *Runner) rememberLease(jobID, token string) {
|
||||
if jobID == "" || token == "" {
|
||||
return
|
||||
}
|
||||
r.leasesMu.Lock()
|
||||
if r.leases == nil {
|
||||
r.leases = make(map[string]string)
|
||||
}
|
||||
r.leases[jobID] = token
|
||||
r.leasesMu.Unlock()
|
||||
}
|
||||
|
||||
func (r *Runner) forgetLease(jobID string) {
|
||||
r.leasesMu.Lock()
|
||||
delete(r.leases, jobID)
|
||||
r.leasesMu.Unlock()
|
||||
}
|
||||
|
||||
func (r *Runner) drainFinalResults() {
|
||||
// Shutdown keeps the existing connection just long enough to report a
|
||||
// bounded set of terminal frames and leases that cannot complete.
|
||||
r.clientMu.Lock()
|
||||
conn, writeMu := r.controlConn, r.controlWriteMu
|
||||
r.clientMu.Unlock()
|
||||
if conn == nil || writeMu == nil {
|
||||
return
|
||||
}
|
||||
deadline := time.Now().Add(finalDrainTimeout)
|
||||
count := 0
|
||||
r.leasesMu.Lock()
|
||||
leases := make(map[string]string, len(r.leases))
|
||||
for id, token := range r.leases {
|
||||
leases[id] = token
|
||||
}
|
||||
r.leasesMu.Unlock()
|
||||
for count < finalDrainLimit && time.Now().Before(deadline) {
|
||||
select {
|
||||
case env := <-r.results:
|
||||
for i := range env.reports {
|
||||
env.reports[i].LeaseToken = env.job.LeaseToken
|
||||
if r.writeControlMessage(conn, writeMu, wire.WorkerMessage{Kind: "result", Result: &env.reports[i]}) != nil {
|
||||
return
|
||||
}
|
||||
r.forgetLease(env.job.JobID)
|
||||
count++
|
||||
if count >= finalDrainLimit {
|
||||
return
|
||||
}
|
||||
}
|
||||
case env := <-r.notifyResults:
|
||||
if r.writeControlMessage(conn, writeMu, wire.WorkerMessage{Kind: "result", NotificationResult: &env.report}) != nil {
|
||||
return
|
||||
}
|
||||
r.forgetLease(env.task.JobID)
|
||||
count++
|
||||
default:
|
||||
for id, token := range leases {
|
||||
if count >= finalDrainLimit || time.Now().After(deadline) {
|
||||
return
|
||||
}
|
||||
r.leasesMu.Lock()
|
||||
_, pending := r.leases[id]
|
||||
r.leasesMu.Unlock()
|
||||
if pending {
|
||||
if r.writeControlMessage(conn, writeMu, wire.WorkerMessage{Kind: "stale_lease", StaleLease: &wire.StaleLeaseReport{JobID: id, LeaseToken: token}}) != nil {
|
||||
return
|
||||
}
|
||||
count++
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (r *Runner) taskInScope(accountID int64) bool {
|
||||
if accountID == 0 {
|
||||
return true
|
||||
} // legacy task payload
|
||||
r.identityMu.RLock()
|
||||
defer r.identityMu.RUnlock()
|
||||
return r.state.AccountID == 0 || r.state.AccountID == accountID
|
||||
}
|
||||
|
||||
func matchingEnvelopeJobID(outer, inner string) bool {
|
||||
return outer != "" && outer == inner
|
||||
}
|
||||
@@ -888,6 +1039,7 @@ func (r *Runner) writer(conn *websocket.Conn, writeMu *sync.Mutex, done <-chan s
|
||||
report.JobID, report.CheckID, env.job.Kind, report.State,
|
||||
)
|
||||
r.resultsBuf.add(resultRowFromReport(&env, report, time.Now().UTC()))
|
||||
r.forgetLease(env.job.JobID)
|
||||
}
|
||||
case env, ok := <-r.notifyResults:
|
||||
if !ok {
|
||||
@@ -907,6 +1059,7 @@ func (r *Runner) writer(conn *websocket.Conn, writeMu *sync.Mutex, done <-chan s
|
||||
"worker: completed notification job=%s method=%s status=%s message=%d duration_ms=%d",
|
||||
env.report.JobID, env.task.Method, env.report.Status, env.task.MessageID, env.report.DurationMs,
|
||||
)
|
||||
r.forgetLease(env.task.JobID)
|
||||
case metric := <-r.metricResults:
|
||||
if metric.generation != metricGeneration {
|
||||
continue
|
||||
@@ -961,7 +1114,10 @@ func (r *Runner) writeControlMessage(conn *websocket.Conn, writeMu *sync.Mutex,
|
||||
return conn.WriteJSON(message)
|
||||
}
|
||||
|
||||
func (r *Runner) applyInit(init *wire.WorkerInit) {
|
||||
func (r *Runner) applyInit(init *wire.WorkerInit) error {
|
||||
if err := r.verifyInit(init); err != nil {
|
||||
return err
|
||||
}
|
||||
if init.Concurrency > 0 && init.Concurrency != r.Concurrency() {
|
||||
size := init.Concurrency
|
||||
if size > r.maxConcurrency {
|
||||
@@ -1053,6 +1209,79 @@ func (r *Runner) applyInit(init *wire.WorkerInit) {
|
||||
"config", init.WorkerID, init.RegionCode, init.Version, init.Capabilities,
|
||||
init.Concurrency, len(init.LLMs), smtpCount, tgCount, len(init.SystemContacts), len(init.Peers),
|
||||
)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *Runner) verifyInit(init *wire.WorkerInit) error {
|
||||
// Legacy control planes do not sign config. Do not turn existing workers
|
||||
// into an outage; once bootstrap pins a key, signed config is mandatory.
|
||||
r.identityMu.Lock()
|
||||
defer r.identityMu.Unlock()
|
||||
if r.state.VerificationKey == "" {
|
||||
return nil
|
||||
}
|
||||
if init.Signature == "" || init.SigningKeyID == "" || init.SigningKeyID != r.state.SigningKeyID || init.AccountID == nil || init.WorkerID != r.state.WorkerID {
|
||||
return fmt.Errorf("unsigned or mismatched config")
|
||||
}
|
||||
if r.state.AccountID != 0 && *init.AccountID != r.state.AccountID {
|
||||
return fmt.Errorf("account scope changed")
|
||||
}
|
||||
if init.ConfigVersion < r.state.ConfigVersion {
|
||||
return fmt.Errorf("config version downgrade")
|
||||
}
|
||||
expires, err := time.Parse(time.RFC3339Nano, init.ExpiresAt)
|
||||
if err != nil || !expires.After(time.Now()) {
|
||||
return fmt.Errorf("config expired")
|
||||
}
|
||||
key, err := base64.StdEncoding.DecodeString(r.state.VerificationKey)
|
||||
if err != nil || len(key) != ed25519.PublicKeySize {
|
||||
return fmt.Errorf("invalid pinned verification key")
|
||||
}
|
||||
sig, err := base64.StdEncoding.DecodeString(init.Signature)
|
||||
if err != nil {
|
||||
return fmt.Errorf("invalid config signature")
|
||||
}
|
||||
signature := init.Signature
|
||||
init.Signature = ""
|
||||
body, err := json.Marshal(init)
|
||||
init.Signature = signature
|
||||
if err != nil || !ed25519.Verify(ed25519.PublicKey(key), body, sig) {
|
||||
return fmt.Errorf("invalid config signature")
|
||||
}
|
||||
// Issued/expiry timestamps are signed freshness metadata, regenerated on
|
||||
// reconnect; they do not change immutable configuration content.
|
||||
canonical := *init
|
||||
canonical.Signature, canonical.IssuedAt, canonical.ExpiresAt = "", "", ""
|
||||
canonicalBody, err := json.Marshal(&canonical)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
payload := base64.StdEncoding.EncodeToString(canonicalBody)
|
||||
if init.ConfigVersion == r.state.ConfigVersion && r.state.ConfigPayload != "" && r.state.ConfigPayload != payload {
|
||||
return fmt.Errorf("config version replay content mismatch")
|
||||
}
|
||||
r.state.AccountID, r.state.ConfigVersion, r.state.ConfigPayload = *init.AccountID, init.ConfigVersion, payload
|
||||
if err := saveWorkerState(r.config.StateFile, r.state); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *Runner) applyRotation(conn *websocket.Conn, writeMu *sync.Mutex, token, rotationID string) {
|
||||
if token == "" || token == r.config.Token {
|
||||
return
|
||||
}
|
||||
r.clientMu.Lock()
|
||||
r.config.Token = token
|
||||
r.client = NewClient(r.config.URL, token)
|
||||
r.state.Token = token
|
||||
r.clientMu.Unlock()
|
||||
if err := saveWorkerState(r.config.StateFile, r.state); err != nil {
|
||||
log.Printf("worker: rotation state: %v", err)
|
||||
return
|
||||
}
|
||||
_ = r.writeControlMessage(conn, writeMu, wire.WorkerMessage{Kind: "rotation_ack", RotationAck: &wire.RotationAck{RotationID: rotationID}})
|
||||
_ = conn.Close()
|
||||
}
|
||||
|
||||
// Credentials returns a snapshot of the current notification credentials.
|
||||
|
||||
@@ -1,8 +1,13 @@
|
||||
package distworker
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/gorilla/websocket"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
@@ -18,6 +23,44 @@ func TestEnqueueTaskMessagePrefersEnvelopeOverLegacyCheck(t *testing.T) {
|
||||
assert.Empty(t, r.jobQueue, "the sibling legacy field must not run a second check")
|
||||
}
|
||||
|
||||
func TestDrainFinalResultsBoundsAndReportsStaleLeases(t *testing.T) {
|
||||
upgrader := websocket.Upgrader{}
|
||||
reports := make(chan wire.WorkerMessage, finalDrainLimit+2)
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
|
||||
conn, err := upgrader.Upgrade(w, req, nil)
|
||||
require.NoError(t, err)
|
||||
defer conn.Close()
|
||||
for {
|
||||
var message wire.WorkerMessage
|
||||
if err := conn.ReadJSON(&message); err != nil {
|
||||
return
|
||||
}
|
||||
reports <- message
|
||||
}
|
||||
}))
|
||||
defer server.Close()
|
||||
wsURL := "ws" + server.URL[len("http"):]
|
||||
conn, _, err := websocket.DefaultDialer.Dial(wsURL, nil)
|
||||
require.NoError(t, err)
|
||||
defer conn.Close()
|
||||
r := &Runner{controlConn: conn, controlWriteMu: &sync.Mutex{}, results: make(chan resultEnvelope, 2), notifyResults: make(chan notifyResultEnvelope, 2), leases: map[string]string{"stale": "lease-stale"}}
|
||||
r.results <- resultEnvelope{job: wire.CheckJob{JobID: "done", LeaseToken: "lease-done"}, reports: []wire.CheckResultReport{{JobID: "done"}}}
|
||||
r.drainFinalResults()
|
||||
found := false
|
||||
deadline := time.After(time.Second)
|
||||
for !found {
|
||||
select {
|
||||
case message := <-reports:
|
||||
if message.StaleLease != nil {
|
||||
assert.Equal(t, "stale", message.StaleLease.JobID)
|
||||
found = true
|
||||
}
|
||||
case <-deadline:
|
||||
t.Fatal("missing stale lease report")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnqueueTaskMessageRejectsMismatchedEnvelopeJobIDs(t *testing.T) {
|
||||
r := &Runner{jobQueue: make(chan wire.CheckJob, 1), results: make(chan resultEnvelope, 1), stopCh: make(chan struct{})}
|
||||
message := wire.WorkerMessage{Kind: "task", TaskEnvelope: &wire.TaskEnvelope{
|
||||
@@ -170,3 +213,11 @@ func TestEnqueueTaskMessageRejectsEmptyLeaseWithoutSideEffects(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnqueueTaskMessageRejectsCrossAccountTask(t *testing.T) {
|
||||
r := &Runner{jobQueue: make(chan wire.CheckJob, 1), results: make(chan resultEnvelope, 1), stopCh: make(chan struct{}), state: workerState{AccountID: 4}}
|
||||
require.True(t, r.enqueueTaskMessage(wire.WorkerMessage{Kind: "task", Task: &wire.CheckJob{JobID: "job", LeaseToken: "lease", AccountID: 5}}))
|
||||
env := <-r.results
|
||||
require.NotNil(t, env.reports[0].Error)
|
||||
assert.Equal(t, "account_scope_mismatch", *env.reports[0].Error)
|
||||
}
|
||||
|
||||
98
internal/distworker/state.go
Обычный файл
98
internal/distworker/state.go
Обычный файл
@@ -0,0 +1,98 @@
|
||||
package distworker
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
)
|
||||
|
||||
type workerState struct {
|
||||
Token string `json:"token"`
|
||||
WorkerID string `json:"worker_id"`
|
||||
VerificationKey string `json:"verification_key"`
|
||||
SigningKeyID string `json:"signing_key_id"`
|
||||
AccountID int64 `json:"account_id"`
|
||||
ConfigVersion int64 `json:"config_version"`
|
||||
ConfigPayload string `json:"config_payload"`
|
||||
}
|
||||
|
||||
func loadWorkerState(path string) (workerState, error) {
|
||||
if path == "" {
|
||||
return workerState{}, nil
|
||||
}
|
||||
if !filepath.IsAbs(path) {
|
||||
return workerState{}, fmt.Errorf("worker state file must be absolute")
|
||||
}
|
||||
if info, err := os.Stat(filepath.Dir(path)); err != nil || !info.IsDir() || info.Mode().Perm() != 0o700 {
|
||||
return workerState{}, fmt.Errorf("worker state directory must have mode 0700")
|
||||
}
|
||||
info, err := os.Stat(path)
|
||||
if os.IsNotExist(err) {
|
||||
return workerState{}, nil
|
||||
}
|
||||
if err != nil {
|
||||
return workerState{}, err
|
||||
}
|
||||
if info.Mode().Perm() != 0o600 {
|
||||
return workerState{}, fmt.Errorf("worker state file must have mode 0600")
|
||||
}
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return workerState{}, err
|
||||
}
|
||||
var state workerState
|
||||
if err := json.Unmarshal(data, &state); err != nil {
|
||||
return workerState{}, err
|
||||
}
|
||||
return state, nil
|
||||
}
|
||||
|
||||
func saveWorkerState(path string, state workerState) error {
|
||||
if path == "" {
|
||||
return nil
|
||||
}
|
||||
if !filepath.IsAbs(path) {
|
||||
return fmt.Errorf("worker state file must be absolute")
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
|
||||
return err
|
||||
}
|
||||
if info, err := os.Stat(filepath.Dir(path)); err != nil || !info.IsDir() || info.Mode().Perm() != 0o700 {
|
||||
return fmt.Errorf("worker state directory must have mode 0700")
|
||||
}
|
||||
data, err := json.Marshal(state)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
tmp, err := os.CreateTemp(filepath.Dir(path), ".state-")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
tmpName := tmp.Name()
|
||||
defer os.Remove(tmpName)
|
||||
if err := tmp.Chmod(0o600); err != nil {
|
||||
_ = tmp.Close()
|
||||
return err
|
||||
}
|
||||
if _, err := tmp.Write(data); err != nil {
|
||||
_ = tmp.Close()
|
||||
return err
|
||||
}
|
||||
if err := tmp.Sync(); err != nil {
|
||||
_ = tmp.Close()
|
||||
return err
|
||||
}
|
||||
if err := tmp.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.Rename(tmpName, path); err != nil {
|
||||
return err
|
||||
}
|
||||
dir, err := os.Open(filepath.Dir(path))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer dir.Close()
|
||||
return dir.Sync()
|
||||
}
|
||||
33
internal/distworker/state_test.go
Обычный файл
33
internal/distworker/state_test.go
Обычный файл
@@ -0,0 +1,33 @@
|
||||
package distworker
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestWorkerStateRoundTripUsesPrivatePermissions(t *testing.T) {
|
||||
dir := filepath.Join(t.TempDir(), "state")
|
||||
require.NoError(t, os.Mkdir(dir, 0o700))
|
||||
path := filepath.Join(dir, "state.json")
|
||||
want := workerState{Token: "secret", WorkerID: "worker-1", VerificationKey: "key", SigningKeyID: "key-2026", AccountID: 7, ConfigVersion: 2}
|
||||
require.NoError(t, saveWorkerState(path, want))
|
||||
info, err := os.Stat(path)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, os.FileMode(0o600), info.Mode().Perm())
|
||||
got, err := loadWorkerState(path)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, want, got)
|
||||
}
|
||||
|
||||
func TestWorkerStateRejectsInsecurePermissions(t *testing.T) {
|
||||
dir := filepath.Join(t.TempDir(), "state")
|
||||
require.NoError(t, os.Mkdir(dir, 0o700))
|
||||
path := filepath.Join(dir, "state.json")
|
||||
require.NoError(t, os.WriteFile(path, []byte(`{"token":"secret"}`), 0o644))
|
||||
_, err := loadWorkerState(path)
|
||||
require.Error(t, err)
|
||||
}
|
||||
63
internal/distworker/trust_test.go
Обычный файл
63
internal/distworker/trust_test.go
Обычный файл
@@ -0,0 +1,63 @@
|
||||
package distworker
|
||||
|
||||
import (
|
||||
"crypto/ed25519"
|
||||
"crypto/rand"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"rocketgit.ru/rsmon/worker/internal/wire"
|
||||
)
|
||||
|
||||
func signedInit(t *testing.T, private ed25519.PrivateKey, account, version int64) *wire.WorkerInit {
|
||||
t.Helper()
|
||||
init := &wire.WorkerInit{WorkerID: "w", SigningKeyID: "v1", AccountID: &account, ConfigVersion: version, ExpiresAt: time.Now().Add(time.Minute).Format(time.RFC3339Nano)}
|
||||
body, err := json.Marshal(init)
|
||||
require.NoError(t, err)
|
||||
init.Signature = base64.StdEncoding.EncodeToString(ed25519.Sign(private, body))
|
||||
return init
|
||||
}
|
||||
|
||||
func TestVerifyInitRejectsTamperWrongKeyAndAccountChange(t *testing.T) {
|
||||
pub, priv, err := ed25519.GenerateKey(rand.Reader)
|
||||
require.NoError(t, err)
|
||||
r := NewRunner(&Config{})
|
||||
r.state = workerState{WorkerID: "w", VerificationKey: base64.StdEncoding.EncodeToString(pub), SigningKeyID: "v1", AccountID: 1}
|
||||
init := signedInit(t, priv, 1, 1)
|
||||
require.NoError(t, r.verifyInit(init))
|
||||
init.ConfigVersion = 2
|
||||
assert.Error(t, r.verifyInit(init), "tamper must invalidate signature")
|
||||
_, other, err := ed25519.GenerateKey(rand.Reader)
|
||||
require.NoError(t, err)
|
||||
assert.Error(t, r.verifyInit(signedInit(t, other, 1, 2)), "wrong key")
|
||||
assert.Error(t, r.verifyInit(signedInit(t, priv, 2, 2)), "account scope must be immutable")
|
||||
unknown := signedInit(t, priv, 1, 2)
|
||||
unknown.SigningKeyID = "unknown"
|
||||
assert.Error(t, r.verifyInit(unknown), "unknown key id")
|
||||
}
|
||||
|
||||
func TestVerifyInitAcceptsIdenticalRestartReplayOnly(t *testing.T) {
|
||||
pub, priv, err := ed25519.GenerateKey(rand.Reader)
|
||||
require.NoError(t, err)
|
||||
first := NewRunner(&Config{})
|
||||
first.state = workerState{WorkerID: "w", VerificationKey: base64.StdEncoding.EncodeToString(pub), SigningKeyID: "v1", AccountID: 1}
|
||||
init := signedInit(t, priv, 1, 4)
|
||||
require.NoError(t, first.verifyInit(init))
|
||||
// Restart restores only durable identity state. The exact verified snapshot
|
||||
// remains safe to replay at the same generation.
|
||||
restarted := NewRunner(&Config{})
|
||||
restarted.state = first.state
|
||||
require.NoError(t, restarted.verifyInit(signedInit(t, priv, 1, 4)))
|
||||
assert.Error(t, restarted.verifyInit(signedInit(t, priv, 1, 3)), "older generation")
|
||||
different := signedInit(t, priv, 1, 4)
|
||||
different.RegionCode = "other"
|
||||
body, err := json.Marshal(&wire.WorkerInit{WorkerID: different.WorkerID, SigningKeyID: different.SigningKeyID, AccountID: different.AccountID, ConfigVersion: different.ConfigVersion, ExpiresAt: different.ExpiresAt, RegionCode: different.RegionCode})
|
||||
require.NoError(t, err)
|
||||
different.Signature = base64.StdEncoding.EncodeToString(ed25519.Sign(priv, body))
|
||||
assert.Error(t, restarted.verifyInit(different), "same generation different content")
|
||||
require.NoError(t, restarted.verifyInit(signedInit(t, priv, 1, 5)), "newer generation")
|
||||
}
|
||||
@@ -18,7 +18,14 @@ import (
|
||||
"golang.org/x/crypto/ssh/knownhosts"
|
||||
)
|
||||
|
||||
type DeployOptions struct {
|
||||
// SSHOptions is the SSH connection and authentication surface shared by
|
||||
// the deploy command and the source installer. Secrets (passwords,
|
||||
// passphrases, sudo passwords) can be supplied through direct flags or
|
||||
// file options. The CLI layer should strongly prefer file options: when
|
||||
// read from a file they never appear in command arguments, logs, or
|
||||
// shell history, while direct flags expose the value through the process
|
||||
// list and shell history.
|
||||
type SSHOptions struct {
|
||||
Host string
|
||||
Port int
|
||||
User string
|
||||
@@ -29,12 +36,16 @@ type DeployOptions struct {
|
||||
KnownHostsFile string
|
||||
HostKeyFingerprint string
|
||||
InsecureHostKey bool
|
||||
Binary string
|
||||
Token string
|
||||
URL string
|
||||
Docker bool
|
||||
Image string
|
||||
NoStart bool
|
||||
}
|
||||
|
||||
type DeployOptions struct {
|
||||
SSHOptions
|
||||
Binary string
|
||||
Token string
|
||||
URL string
|
||||
Docker bool
|
||||
Image string
|
||||
NoStart bool
|
||||
}
|
||||
|
||||
func Deploy(opts DeployOptions) error {
|
||||
@@ -71,11 +82,11 @@ func Deploy(opts DeployOptions) error {
|
||||
return err
|
||||
}
|
||||
}
|
||||
auth, err := sshAuth(opts)
|
||||
auth, err := sshAuth(opts.SSHOptions)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
hostKey, err := hostKeyCallback(opts)
|
||||
hostKey, err := hostKeyCallback(opts.SSHOptions)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -111,16 +122,7 @@ func Deploy(opts DeployOptions) error {
|
||||
if opts.NoStart {
|
||||
args += " --no-start"
|
||||
}
|
||||
var command string
|
||||
var stdin []byte
|
||||
if opts.User == "root" {
|
||||
command = args
|
||||
} else if opts.SudoPassword != "" {
|
||||
command = "sudo -S -p '' -- " + args
|
||||
stdin = []byte(opts.SudoPassword + "\n")
|
||||
} else {
|
||||
command = "sudo -n -- " + args
|
||||
}
|
||||
command, stdin := sudoWrap(opts.User, opts.SudoPassword, args)
|
||||
if err := runRemote(client, command, stdin); err != nil {
|
||||
return fmt.Errorf("remote install: %w", err)
|
||||
}
|
||||
@@ -137,23 +139,30 @@ func deployDocker(client *ssh.Client, opts DeployOptions, remoteEnv, remoteUnit
|
||||
if !opts.NoStart {
|
||||
script += " && systemctl restart rsmon-worker.service && systemctl is-active --quiet rsmon-worker.service"
|
||||
}
|
||||
command := "sh -c " + shellQuote(script)
|
||||
var stdin []byte
|
||||
if opts.User != "root" {
|
||||
if opts.SudoPassword != "" {
|
||||
command = "sudo -S -p '' -- " + command
|
||||
stdin = []byte(opts.SudoPassword + "\n")
|
||||
} else {
|
||||
command = "sudo -n -- " + command
|
||||
}
|
||||
}
|
||||
command, stdin := sudoWrap(opts.User, opts.SudoPassword, "sh -c "+shellQuote(script))
|
||||
if err := runRemote(client, command, stdin); err != nil {
|
||||
return fmt.Errorf("remote Docker install: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func sshAuth(opts DeployOptions) ([]ssh.AuthMethod, error) {
|
||||
// sudoWrap prefixes a remote command with the privilege path required to
|
||||
// run it as root: the plain command for a root user, `sudo -n` when the
|
||||
// user has passwordless sudo, and `sudo -S` with an empty prompt when a
|
||||
// sudo password is configured. The sudo password is delivered only over
|
||||
// the session's stdin, never in the command string, so it cannot appear
|
||||
// in process listings, logs, or shell history.
|
||||
func sudoWrap(user, sudoPassword, command string) (string, []byte) {
|
||||
if user == "root" {
|
||||
return command, nil
|
||||
}
|
||||
if sudoPassword != "" {
|
||||
return "sudo -S -p '' -- " + command, []byte(sudoPassword + "\n")
|
||||
}
|
||||
return "sudo -n -- " + command, nil
|
||||
}
|
||||
|
||||
func sshAuth(opts SSHOptions) ([]ssh.AuthMethod, error) {
|
||||
var methods []ssh.AuthMethod
|
||||
if opts.IdentityFile != "" {
|
||||
key, err := os.ReadFile(opts.IdentityFile)
|
||||
@@ -180,7 +189,7 @@ func sshAuth(opts DeployOptions) ([]ssh.AuthMethod, error) {
|
||||
return methods, nil
|
||||
}
|
||||
|
||||
func hostKeyCallback(opts DeployOptions) (ssh.HostKeyCallback, error) {
|
||||
func hostKeyCallback(opts SSHOptions) (ssh.HostKeyCallback, error) {
|
||||
if opts.HostKeyFingerprint != "" {
|
||||
want := opts.HostKeyFingerprint
|
||||
return func(_ string, _ net.Addr, key ssh.PublicKey) error {
|
||||
@@ -242,6 +251,79 @@ func uploadReader(client *ssh.Client, src io.Reader, remotePath string, mode os.
|
||||
}
|
||||
|
||||
func runRemote(client *ssh.Client, command string, stdin []byte) error {
|
||||
// Deploy commands are short and stream to the terminal; they keep the
|
||||
// historical behavior with no timeout.
|
||||
return runRemoteWithTimeout(client, command, stdin, os.Stdout, os.Stderr, 0)
|
||||
}
|
||||
|
||||
// maxRemoteError bounds the stderr snippet folded into runRemoteOutput
|
||||
// errors so a verbose remote failure cannot produce an unbounded error
|
||||
// string.
|
||||
const maxRemoteError = 4096
|
||||
|
||||
// maxRemoteOutput bounds the stdout captured by runRemoteOutput so a
|
||||
// noisy remote command cannot exhaust memory.
|
||||
const maxRemoteOutput = 1 << 20 // 1 MiB
|
||||
|
||||
// boundedBuffer is an io.Writer that silently discards everything past
|
||||
// max bytes and remembers whether truncation happened.
|
||||
type boundedBuffer struct {
|
||||
buf bytes.Buffer
|
||||
max int
|
||||
truncated bool
|
||||
}
|
||||
|
||||
func (b *boundedBuffer) Write(p []byte) (int, error) {
|
||||
if len(p) == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
if b.buf.Len() >= b.max {
|
||||
b.truncated = true
|
||||
return len(p), nil
|
||||
}
|
||||
remaining := b.max - b.buf.Len()
|
||||
if len(p) > remaining {
|
||||
b.buf.Write(p[:remaining])
|
||||
b.truncated = true
|
||||
return len(p), nil
|
||||
}
|
||||
return b.buf.Write(p)
|
||||
}
|
||||
|
||||
func (b *boundedBuffer) Bytes() []byte { return b.buf.Bytes() }
|
||||
|
||||
func (b *boundedBuffer) String() string { return b.buf.String() }
|
||||
|
||||
// runRemoteOutput executes a remote command and returns its captured,
|
||||
// size-bounded stdout. Stderr is folded into the returned error on
|
||||
// failure (bounded to maxRemoteError bytes) so operators see what went
|
||||
// wrong without a bounded failure dumping unbounded output. The command
|
||||
// is aborted if it outlives timeout (<= 0 disables the timeout).
|
||||
func runRemoteOutput(client *ssh.Client, command string, stdin []byte, timeout time.Duration) ([]byte, error) {
|
||||
var stdout, stderr boundedBuffer
|
||||
stdout.max = maxRemoteOutput
|
||||
stderr.max = maxRemoteError
|
||||
if err := runRemoteWithTimeout(client, command, stdin, &stdout, &stderr, timeout); err != nil {
|
||||
msg := strings.TrimSpace(stderr.String())
|
||||
if msg != "" {
|
||||
if stderr.truncated {
|
||||
msg += "..."
|
||||
}
|
||||
return stdout.Bytes(), fmt.Errorf("%w: %s", err, msg)
|
||||
}
|
||||
return stdout.Bytes(), err
|
||||
}
|
||||
out := stdout.Bytes()
|
||||
if stdout.truncated {
|
||||
out = append(out, []byte("\n...[output truncated]")...)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// runRemoteWithTimeout runs a remote command, optionally aborting it
|
||||
// when it outlives timeout (<= 0 disables the timeout). The session is
|
||||
// closed and the blocked Run is unblocked when the timer fires.
|
||||
func runRemoteWithTimeout(client *ssh.Client, command string, stdin []byte, stdout, stderr io.Writer, timeout time.Duration) error {
|
||||
session, err := client.NewSession()
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -250,9 +332,24 @@ func runRemote(client *ssh.Client, command string, stdin []byte) error {
|
||||
if stdin != nil {
|
||||
session.Stdin = bytes.NewReader(stdin)
|
||||
}
|
||||
session.Stdout = os.Stdout
|
||||
session.Stderr = os.Stderr
|
||||
return session.Run(command)
|
||||
session.Stdout = stdout
|
||||
session.Stderr = stderr
|
||||
|
||||
if timeout <= 0 {
|
||||
return session.Run(command)
|
||||
}
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- session.Run(command) }()
|
||||
timer := time.NewTimer(timeout)
|
||||
defer timer.Stop()
|
||||
select {
|
||||
case err := <-done:
|
||||
return err
|
||||
case <-timer.C:
|
||||
_ = session.Close() // abort the remote command and free the session
|
||||
<-done
|
||||
return fmt.Errorf("remote command timed out after %s", timeout)
|
||||
}
|
||||
}
|
||||
|
||||
func shellQuote(value string) string {
|
||||
|
||||
@@ -19,14 +19,14 @@ func TestFingerprintHostKeyCallback(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
callback, err := hostKeyCallback(DeployOptions{HostKeyFingerprint: ssh.FingerprintSHA256(publicKey)})
|
||||
callback, err := hostKeyCallback(SSHOptions{HostKeyFingerprint: ssh.FingerprintSHA256(publicKey)})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := callback("host", &net.TCPAddr{}, publicKey); err != nil {
|
||||
t.Fatalf("matching fingerprint rejected: %v", err)
|
||||
}
|
||||
callback, err = hostKeyCallback(DeployOptions{HostKeyFingerprint: "SHA256:wrong"})
|
||||
callback, err = hostKeyCallback(SSHOptions{HostKeyFingerprint: "SHA256:wrong"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -36,15 +36,17 @@ func TestFingerprintHostKeyCallback(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestKnownHostsMissingFile(t *testing.T) {
|
||||
if _, err := hostKeyCallback(DeployOptions{KnownHostsFile: t.TempDir() + "/missing"}); err == nil {
|
||||
if _, err := hostKeyCallback(SSHOptions{KnownHostsFile: t.TempDir() + "/missing"}); err == nil {
|
||||
t.Fatal("missing known_hosts file accepted")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeployRejectsMutableDockerImageBeforeConnecting(t *testing.T) {
|
||||
err := Deploy(DeployOptions{
|
||||
Host: "unreachable.example.test",
|
||||
User: "deploy",
|
||||
SSHOptions: SSHOptions{
|
||||
Host: "unreachable.example.test",
|
||||
User: "deploy",
|
||||
},
|
||||
Token: "token",
|
||||
Docker: true,
|
||||
Image: "reg.rsxx.ru/rsmon/rsmon-worker:latest",
|
||||
|
||||
621
internal/installer/harness/harness.go
Обычный файл
621
internal/installer/harness/harness.go
Обычный файл
@@ -0,0 +1,621 @@
|
||||
// Package harness starts real OpenSSH containers for the source-install
|
||||
// tests. It is the reusable Docker/OpenSSH test harness from work
|
||||
// package 1 of docs/source-installation.md.
|
||||
//
|
||||
// The harness never mocks SSH: it builds a distro fixture image, runs an
|
||||
// OpenSSH server in a container, waits for real network readiness, and
|
||||
// connects with the golang.org/x/crypto/ssh library and known_hosts
|
||||
// verification semantics the worker installer uses. Tests opt in with
|
||||
// RSMON_TEST_DOCKER=1 so ordinary unit runs never pull or start Docker.
|
||||
package harness
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"golang.org/x/crypto/ssh"
|
||||
"golang.org/x/crypto/ssh/knownhosts"
|
||||
|
||||
"rocketgit.ru/rsmon/worker/internal/sshinstall"
|
||||
)
|
||||
|
||||
// Fixture describes one distro OpenSSH fixture the harness can start.
|
||||
type Fixture struct {
|
||||
Name string // "alpine", "ubuntu", "arch"
|
||||
Distro sshinstall.Distro // expected detected distro
|
||||
Pkg sshinstall.PackageManager
|
||||
Init sshinstall.InitSystem
|
||||
Image string // explicit image override; empty uses the default
|
||||
User string // SSH user to authenticate as; default "root"
|
||||
}
|
||||
|
||||
// Fixtures returns the supported source-install distro fixtures.
|
||||
func Fixtures() []Fixture {
|
||||
return []Fixture{
|
||||
{Name: "alpine", Distro: sshinstall.DistroAlpine, Pkg: sshinstall.PkgApk, Init: sshinstall.InitOpenRC},
|
||||
{Name: "ubuntu", Distro: sshinstall.DistroUbuntu, Pkg: sshinstall.PkgApt, Init: sshinstall.InitSystemd},
|
||||
{Name: "arch", Distro: sshinstall.DistroArch, Pkg: sshinstall.PkgPacman, Init: sshinstall.InitSystemd},
|
||||
}
|
||||
}
|
||||
|
||||
// defaultImages maps fixture name to its base image. reg.rsxx.ru mirror
|
||||
// refs are used where the mirror caches the distro; Docker Hub refs are
|
||||
// the fallback for distros the mirror does not carry and are overridable
|
||||
// per fixture via RSMON_TEST_IMAGE_<NAME>.
|
||||
var defaultImages = map[string]string{
|
||||
"alpine": "reg.rsxx.ru/library/alpine:3",
|
||||
"ubuntu": "ubuntu:24.04",
|
||||
"arch": "archlinux:latest",
|
||||
}
|
||||
|
||||
// ImageRef resolves the base image reference for a fixture. Precedence:
|
||||
// RSMON_TEST_IMAGE_<NAME> env > Fixture.Image > mirror-aware default.
|
||||
func (f Fixture) ImageRef() string {
|
||||
envKey := "RSMON_TEST_IMAGE_" + strings.ToUpper(strings.ReplaceAll(f.Name, "-", "_"))
|
||||
if v := strings.TrimSpace(os.Getenv(envKey)); v != "" {
|
||||
return v
|
||||
}
|
||||
if f.Image != "" {
|
||||
return f.Image
|
||||
}
|
||||
return defaultImages[f.Name]
|
||||
}
|
||||
|
||||
// UserOrDefault returns the SSH user, defaulting to root.
|
||||
func (f Fixture) UserOrDefault() string {
|
||||
if f.User != "" {
|
||||
return f.User
|
||||
}
|
||||
return "root"
|
||||
}
|
||||
|
||||
// DockerfilePath is the fixture's Dockerfile under the package testdata.
|
||||
func (f Fixture) DockerfilePath() string {
|
||||
return filepath.Join("testdata", "fixtures", f.Name, "Dockerfile")
|
||||
}
|
||||
|
||||
// BuildContext is the Docker build context that carries the shared keys.
|
||||
func (f Fixture) BuildContext() string {
|
||||
return filepath.Join("testdata")
|
||||
}
|
||||
|
||||
// Harness manages the lifecycle of one OpenSSH test container: build,
|
||||
// run, wait for readiness, capture the host key, and tear down reliably.
|
||||
// Every resource it creates - fixture image tag, container, network, and
|
||||
// the temp known_hosts directory - is unique to this instance and is
|
||||
// removed by Stop.
|
||||
type Harness struct {
|
||||
Name string
|
||||
Fixture Fixture
|
||||
|
||||
mu sync.Mutex
|
||||
started bool
|
||||
suffix string
|
||||
containerID string
|
||||
container string
|
||||
network string
|
||||
imageTag string
|
||||
port int
|
||||
user string
|
||||
keyPath string
|
||||
knownHosts string
|
||||
}
|
||||
|
||||
// New creates a harness for the named fixture. The name must be a safe,
|
||||
// short identifier; a random suffix makes the container, network, and
|
||||
// fixture-image tag unique to this instance so teardown never touches
|
||||
// another harness's resources.
|
||||
func New(name string, f Fixture) (*Harness, error) {
|
||||
if err := validateName(name); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
suffix := strconv.FormatInt(time.Now().UnixNano(), 36)
|
||||
return &Harness{
|
||||
Name: name,
|
||||
Fixture: f,
|
||||
suffix: suffix,
|
||||
container: "rsmon-worker-test-" + sanitize(name) + "-" + suffix,
|
||||
network: "rsmon-worker-test-" + sanitize(name) + "-" + suffix,
|
||||
imageTag: "rsmon-worker-test/" + sanitize(f.Name) + "-" + suffix + ":local",
|
||||
user: f.UserOrDefault(),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// ContainerID returns the running container id (after Start).
|
||||
func (h *Harness) ContainerID() string {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
return h.containerID
|
||||
}
|
||||
|
||||
// ContainerName returns the Docker container name.
|
||||
func (h *Harness) ContainerName() string { return h.container }
|
||||
|
||||
// NetworkName returns the Docker network name.
|
||||
func (h *Harness) NetworkName() string { return h.network }
|
||||
|
||||
// ImageTag returns the fixture image tag this harness instance builds
|
||||
// and removes on Stop. Tags are unique per instance, so removing one
|
||||
// never deletes a shared base image or another harness's image.
|
||||
func (h *Harness) ImageTag() string { return h.imageTag }
|
||||
|
||||
// KnownHostsPath returns the temp known_hosts file created by Start, or
|
||||
// "" before Start. The surrounding directory is removed by Stop.
|
||||
func (h *Harness) KnownHostsPath() string {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
return h.knownHosts
|
||||
}
|
||||
|
||||
// Addr returns the dialable host:port of the published SSH listener.
|
||||
// Before Start it is empty.
|
||||
func (h *Harness) Addr() string {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
if h.port == 0 {
|
||||
return ""
|
||||
}
|
||||
return net.JoinHostPort("127.0.0.1", strconv.Itoa(h.port))
|
||||
}
|
||||
|
||||
// Port returns the published host port of the container's SSH listener
|
||||
// (the host is always 127.0.0.1). 0 before Start.
|
||||
func (h *Harness) Port() int {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
return h.port
|
||||
}
|
||||
|
||||
// Start builds the fixture image, starts the container, waits for real
|
||||
// SSH readiness, captures the server host key into a temp known_hosts
|
||||
// file, and records the published port. Every error path cleans up the
|
||||
// container, network, image tag, and temp dir created so far; on
|
||||
// success the caller owns teardown via t.Cleanup(h.Stop) or a defer.
|
||||
func (h *Harness) Start(ctx context.Context) error {
|
||||
if !Enabled() {
|
||||
return errors.New("harness integration is disabled; set RSMON_TEST_DOCKER=1 to run Docker/OpenSSH tests")
|
||||
}
|
||||
image := h.Fixture.ImageRef()
|
||||
if image == "" {
|
||||
return fmt.Errorf("fixture %q has no base image", h.Fixture.Name)
|
||||
}
|
||||
|
||||
// Any error after the first resource is created must release what
|
||||
// was already allocated. Stop is idempotent and tolerates resources
|
||||
// that were never created.
|
||||
success := false
|
||||
defer func() {
|
||||
if !success {
|
||||
_ = h.Stop()
|
||||
}
|
||||
}()
|
||||
|
||||
if _, err := dockerCmd(ctx, "build", "-q", "-t", h.imageTag, "-f", h.Fixture.DockerfilePath(), h.Fixture.BuildContext()); err != nil {
|
||||
return fmt.Errorf("build %s fixture image: %w", h.Fixture.Name, err)
|
||||
}
|
||||
if _, err := dockerCmd(ctx, "network", "create", h.network); err != nil {
|
||||
return fmt.Errorf("create test network: %w", err)
|
||||
}
|
||||
// docker run -d prints the container id directly, so no lookup is
|
||||
// needed; the container name is the stable handle for later docker
|
||||
// calls and the id is captured for diagnostics and assertions.
|
||||
runArgs := []string{"run", "-d", "--name", h.container, "--network", h.network, "-p", "127.0.0.1::22"}
|
||||
runArgs = append(runArgs, dockerDNS()...)
|
||||
runArgs = append(runArgs, h.imageTag)
|
||||
out, err := dockerCmd(ctx, runArgs...)
|
||||
if err != nil {
|
||||
return fmt.Errorf("start %s fixture container: %w", h.Fixture.Name, err)
|
||||
}
|
||||
h.containerID = strings.TrimSpace(out)
|
||||
|
||||
port, err := h.publishedPort(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
h.port = port
|
||||
|
||||
key, err := h.waitForSSH(ctx, h.Addr())
|
||||
if err != nil {
|
||||
return fmt.Errorf("wait for ssh readiness on %s fixture: %w", h.Fixture.Name, err)
|
||||
}
|
||||
// Register the temp dir before writing so a write failure still
|
||||
// leaves it known to the cleanup defer.
|
||||
dir, err := os.MkdirTemp("", "rsmon-worker-harness-")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
h.knownHosts = filepath.Join(dir, "known_hosts")
|
||||
hostsFile, err := writeKnownHosts(dir, h.Addr(), key)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
h.keyPath = testKeyPath()
|
||||
h.knownHosts = hostsFile
|
||||
|
||||
h.mu.Lock()
|
||||
h.started = true
|
||||
h.mu.Unlock()
|
||||
success = true
|
||||
return nil
|
||||
}
|
||||
|
||||
// dockerDNS returns the `--dns` arguments to pin for fixture containers,
|
||||
// parsed from the comma-separated RSMON_TEST_DOCKER_DNS environment
|
||||
// variable. It is empty by default (Docker's embedded DNS). The override
|
||||
// exists so environments with flaky local resolvers can pin a reliable
|
||||
// upstream for the internet-facing installs (go.dev, rocketgit.ru,
|
||||
// proxy.golang.org), which would otherwise fail intermittently on DNS
|
||||
// timeouts.
|
||||
func dockerDNS() []string {
|
||||
var args []string
|
||||
for _, ns := range strings.Split(os.Getenv("RSMON_TEST_DOCKER_DNS"), ",") {
|
||||
if ns = strings.TrimSpace(ns); ns != "" {
|
||||
args = append(args, "--dns", ns)
|
||||
}
|
||||
}
|
||||
return args
|
||||
}
|
||||
|
||||
// Stop releases every resource the harness created: the container, its
|
||||
// dedicated network, the per-instance fixture image tag (never a shared
|
||||
// base image), and the temp known_hosts directory. It is idempotent and
|
||||
// tolerates already-removed resources so teardown never fails the test
|
||||
// twice.
|
||||
func (h *Harness) Stop() error {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
|
||||
defer cancel()
|
||||
var errs []error
|
||||
if h.container != "" {
|
||||
if _, err := dockerCmd(ctx, "rm", "-f", h.container); err != nil && !isNotExist(err) {
|
||||
errs = append(errs, err)
|
||||
}
|
||||
}
|
||||
if h.network != "" {
|
||||
if _, err := dockerCmd(ctx, "network", "rm", h.network); err != nil && !isNotExist(err) {
|
||||
errs = append(errs, err)
|
||||
}
|
||||
}
|
||||
if h.imageTag != "" {
|
||||
if _, err := dockerCmd(ctx, "image", "rm", h.imageTag); err != nil && !isNotExist(err) && !isInUse(err) {
|
||||
errs = append(errs, err)
|
||||
}
|
||||
}
|
||||
if h.knownHosts != "" {
|
||||
if err := os.RemoveAll(filepath.Dir(h.knownHosts)); err != nil {
|
||||
errs = append(errs, fmt.Errorf("remove harness temp dir: %w", err))
|
||||
}
|
||||
}
|
||||
h.mu.Lock()
|
||||
h.started = false
|
||||
h.containerID = ""
|
||||
h.port = 0
|
||||
h.keyPath = ""
|
||||
h.knownHosts = ""
|
||||
h.mu.Unlock()
|
||||
return errors.Join(errs...)
|
||||
}
|
||||
|
||||
// Dial connects using the same SSH library (golang.org/x/crypto/ssh) and
|
||||
// the same known_hosts host-key verification the installer's deploy path
|
||||
// relies on. The harness owns this code rather than calling into the
|
||||
// installer package so tests stay independent; only the library and the
|
||||
// verification semantics are shared.
|
||||
func (h *Harness) Dial() (*ssh.Client, error) {
|
||||
h.mu.Lock()
|
||||
if !h.started || h.knownHosts == "" || h.keyPath == "" {
|
||||
h.mu.Unlock()
|
||||
return nil, errors.New("harness is not started")
|
||||
}
|
||||
addr := net.JoinHostPort("127.0.0.1", strconv.Itoa(h.port))
|
||||
knownHostsFile, keyPath, user := h.knownHosts, h.keyPath, h.user
|
||||
h.mu.Unlock()
|
||||
|
||||
keyBytes, err := os.ReadFile(keyPath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
signer, err := ssh.ParsePrivateKey(keyBytes)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
hostKeyCallback, err := knownhosts.New(knownHostsFile)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
config := &ssh.ClientConfig{
|
||||
User: user,
|
||||
Auth: []ssh.AuthMethod{ssh.PublicKeys(signer)},
|
||||
HostKeyCallback: hostKeyCallback,
|
||||
Timeout: 15 * time.Second,
|
||||
}
|
||||
return ssh.Dial("tcp", addr, config)
|
||||
}
|
||||
|
||||
// RunCommand executes a command over an existing SSH client and returns
|
||||
// its stdout. Stderr is folded into the returned error message on
|
||||
// failure so test failures show what went wrong.
|
||||
func RunCommand(client *ssh.Client, command string) ([]byte, error) {
|
||||
session, err := client.NewSession()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer session.Close() //nolint:errcheck
|
||||
var stdout, stderr bytes.Buffer
|
||||
session.Stdout = &stdout
|
||||
session.Stderr = &stderr
|
||||
if err := session.Run(command); err != nil {
|
||||
if msg := strings.TrimSpace(stderr.String()); msg != "" {
|
||||
return stdout.Bytes(), fmt.Errorf("%w: %s", err, msg)
|
||||
}
|
||||
return stdout.Bytes(), err
|
||||
}
|
||||
return stdout.Bytes(), nil
|
||||
}
|
||||
|
||||
// DockerCmd runs the Docker CLI with the given arguments, returning
|
||||
// stdout. Exposed so integration tests can assert teardown state.
|
||||
func (h *Harness) DockerCmd(ctx context.Context, args ...string) (string, error) {
|
||||
return dockerCmd(ctx, args...)
|
||||
}
|
||||
|
||||
// Enabled reports whether Docker-backed integration tests may run. The
|
||||
// opt-in env var keeps ordinary `go test` and CI unit runs from pulling
|
||||
// or starting any container.
|
||||
func Enabled() bool {
|
||||
switch strings.ToLower(strings.TrimSpace(os.Getenv("RSMON_TEST_DOCKER"))) {
|
||||
case "1", "true", "yes", "on":
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// SkipUnlessEnabled skips the test with a helpful message unless the
|
||||
// Docker integration opt-in is set.
|
||||
func SkipUnlessEnabled(t interface {
|
||||
Helper()
|
||||
Skipf(format string, args ...interface{})
|
||||
},
|
||||
) {
|
||||
t.Helper()
|
||||
if !Enabled() {
|
||||
t.Skipf("Docker/OpenSSH harness tests are opt-in; set RSMON_TEST_DOCKER=1 to run them")
|
||||
}
|
||||
}
|
||||
|
||||
// dockerBin is the Docker CLI binary. Overridable in tests via
|
||||
// SetDockerBin so unit tests can stub the harness's docker calls.
|
||||
var dockerBin = "docker"
|
||||
|
||||
// SetDockerBin overrides the Docker CLI binary used by the harness.
|
||||
// Pass an empty value to restore the default ("docker").
|
||||
func SetDockerBin(name string) {
|
||||
if name == "" {
|
||||
dockerBin = "docker"
|
||||
return
|
||||
}
|
||||
dockerBin = name
|
||||
}
|
||||
|
||||
func dockerCmd(ctx context.Context, args ...string) (string, error) {
|
||||
cmd := exec.CommandContext(ctx, dockerBin, args...)
|
||||
out, err := cmd.Output()
|
||||
if err != nil {
|
||||
if ee, ok := err.(*exec.ExitError); ok && len(ee.Stderr) > 0 {
|
||||
return "", fmt.Errorf("docker %s: %s", strings.Join(args, " "), strings.TrimSpace(string(ee.Stderr)))
|
||||
}
|
||||
return "", fmt.Errorf("docker %s: %w", strings.Join(args, " "), err)
|
||||
}
|
||||
return string(out), nil
|
||||
}
|
||||
|
||||
// isNotExist reports whether the docker error is a missing resource
|
||||
// (container/network/image already removed or never created). Docker
|
||||
// reports missing networks and images as "... not found" and missing
|
||||
// containers as "No such container: ...".
|
||||
func isNotExist(err error) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
lower := strings.ToLower(err.Error())
|
||||
for _, marker := range []string{"no such container", "no such network", "no such image", "is not running", "not found"} {
|
||||
if strings.Contains(lower, marker) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// isInUse reports whether a docker image removal failed because another
|
||||
// container or tag still references the image. Teardown must not treat
|
||||
// that as a leak error: the per-instance tags make this unlikely, but
|
||||
// tolerating it keeps Stop deterministic under concurrent harnesses.
|
||||
func isInUse(err error) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
lower := strings.ToLower(err.Error())
|
||||
for _, marker := range []string{"image is being used", "image is in use", "image is referenced"} {
|
||||
if strings.Contains(lower, marker) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// publishedPort queries the host port docker published for the
|
||||
// container's port 22.
|
||||
func (h *Harness) publishedPort(ctx context.Context) (int, error) {
|
||||
out, err := dockerCmd(ctx, "port", h.container, "22/tcp")
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return parsePublishedPort(out)
|
||||
}
|
||||
|
||||
// parsePublishedPort extracts the host port from `docker port` output
|
||||
// like "127.0.0.1:49153", "0.0.0.0:49153", or "::1:49153".
|
||||
func parsePublishedPort(out string) (int, error) {
|
||||
for _, line := range strings.Split(out, "\n") {
|
||||
line = strings.TrimSpace(line)
|
||||
if line == "" {
|
||||
continue
|
||||
}
|
||||
sep := strings.LastIndex(line, ":")
|
||||
if sep < 0 {
|
||||
continue
|
||||
}
|
||||
port, err := strconv.Atoi(strings.TrimSpace(line[sep+1:]))
|
||||
if err == nil && port > 0 && port <= 65535 {
|
||||
return port, nil
|
||||
}
|
||||
}
|
||||
return 0, fmt.Errorf("could not parse published port from %q", out)
|
||||
}
|
||||
|
||||
// waitForSSH waits for the container's SSH listener to accept a real
|
||||
// handshake and returns the server host key. TCP readiness alone is not
|
||||
// enough; the ssh.Dial must succeed and the exec channel must answer.
|
||||
func (h *Harness) waitForSSH(ctx context.Context, addr string) (ssh.PublicKey, error) {
|
||||
if err := waitForPort(ctx, addr, 60*time.Second); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
deadline := time.Now().Add(60 * time.Second)
|
||||
var lastErr error
|
||||
for {
|
||||
if ctx.Err() != nil {
|
||||
return nil, fmt.Errorf("ssh readiness: %w (last: %v)", ctx.Err(), lastErr)
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
return nil, fmt.Errorf("ssh readiness timed out (last: %v)", lastErr)
|
||||
}
|
||||
key, err := h.handshake(addr)
|
||||
if err == nil {
|
||||
return key, nil
|
||||
}
|
||||
lastErr = err
|
||||
time.Sleep(500 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
|
||||
// handshake performs one full SSH exchange and captures the server host
|
||||
// key via the callback so the harness can write a known_hosts entry.
|
||||
// A fresh known_hosts file trusts the first key it sees (trust-on-first-
|
||||
// use); the host-key-mismatch integration test proves that a known_hosts
|
||||
// entry carrying a *different* key is rejected before any command runs.
|
||||
func (h *Harness) handshake(addr string) (ssh.PublicKey, error) {
|
||||
keyBytes, err := os.ReadFile(testKeyPath())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
signer, err := ssh.ParsePrivateKey(keyBytes)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var got ssh.PublicKey
|
||||
config := &ssh.ClientConfig{
|
||||
User: h.user,
|
||||
Auth: []ssh.AuthMethod{ssh.PublicKeys(signer)},
|
||||
HostKeyCallback: func(_ string, _ net.Addr, key ssh.PublicKey) error {
|
||||
got = key
|
||||
return nil
|
||||
},
|
||||
Timeout: 5 * time.Second,
|
||||
}
|
||||
client, err := ssh.Dial("tcp", addr, config)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer client.Close() //nolint:errcheck
|
||||
session, err := client.NewSession()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer session.Close() //nolint:errcheck
|
||||
if _, err := session.Output("true"); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if got == nil {
|
||||
return nil, errors.New("no host key returned by handshake")
|
||||
}
|
||||
return got, nil
|
||||
}
|
||||
|
||||
// waitForPort polls a TCP endpoint until it accepts a connection. It is
|
||||
// a plain network probe; SSH readiness is separately verified.
|
||||
func waitForPort(ctx context.Context, addr string, timeout time.Duration) error {
|
||||
deadline := time.Now().Add(timeout)
|
||||
for {
|
||||
if ctx.Err() != nil {
|
||||
return ctx.Err()
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
return fmt.Errorf("port %s not open within %s", addr, timeout)
|
||||
}
|
||||
conn, err := net.DialTimeout("tcp", addr, 2*time.Second)
|
||||
if err == nil {
|
||||
_ = conn.Close()
|
||||
return nil
|
||||
}
|
||||
time.Sleep(500 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
|
||||
// writeKnownHosts writes a known_hosts entry for the harness address
|
||||
// inside an existing temp directory, using the captured server host key.
|
||||
func writeKnownHosts(dir, addr string, key ssh.PublicKey) (string, error) {
|
||||
path := filepath.Join(dir, "known_hosts")
|
||||
line := knownhosts.Line([]string{addr}, key)
|
||||
if err := os.WriteFile(path, []byte(line+"\n"), 0o600); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return path, nil
|
||||
}
|
||||
|
||||
// testKeyPath returns the shared test private key bundled with the
|
||||
// package. The fixtures bake the matching public key into authorized_keys.
|
||||
//
|
||||
// This keypair is strictly test-only: it grants root SSH access only to
|
||||
// the disposable fixture containers that bake its public key. It must
|
||||
// never be used for real hosts, copied into production images, or
|
||||
// treated as a credential anywhere outside the harness.
|
||||
func testKeyPath() string {
|
||||
return filepath.Join("testdata", "keys", "rsmon_test_ed25519")
|
||||
}
|
||||
|
||||
// validateName rejects harness names that could inject shell or docker
|
||||
// metacharacters into container/network names.
|
||||
func validateName(name string) error {
|
||||
if name == "" || len(name) > 64 {
|
||||
return errors.New("harness name must be 1-64 characters")
|
||||
}
|
||||
for _, r := range name {
|
||||
lower := r >= 'a' && r <= 'z'
|
||||
digit := r >= '0' && r <= '9'
|
||||
if !lower && !digit && r != '-' {
|
||||
return fmt.Errorf("harness name %q must be lowercase alphanumeric and hyphens", name)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func sanitize(name string) string {
|
||||
var b strings.Builder
|
||||
for _, r := range name {
|
||||
if r >= 'a' && r <= 'z' || r >= '0' && r <= '9' {
|
||||
b.WriteRune(r)
|
||||
} else {
|
||||
b.WriteByte('-')
|
||||
}
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
417
internal/installer/harness/harness_test.go
Обычный файл
417
internal/installer/harness/harness_test.go
Обычный файл
@@ -0,0 +1,417 @@
|
||||
package harness
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"golang.org/x/crypto/ssh"
|
||||
)
|
||||
|
||||
func TestEnabled(t *testing.T) {
|
||||
// os.Unsetenv would leak into later tests in the same process;
|
||||
// t.Setenv restores the original value after this test.
|
||||
t.Setenv("RSMON_TEST_DOCKER", "")
|
||||
if Enabled() {
|
||||
t.Fatal("Enabled() true without RSMON_TEST_DOCKER")
|
||||
}
|
||||
for _, v := range []string{"1", "true", "TRUE", "yes", "on"} {
|
||||
t.Setenv("RSMON_TEST_DOCKER", v)
|
||||
if !Enabled() {
|
||||
t.Fatalf("Enabled() false for RSMON_TEST_DOCKER=%q", v)
|
||||
}
|
||||
}
|
||||
for _, v := range []string{"0", "false", "no", "off", " "} {
|
||||
t.Setenv("RSMON_TEST_DOCKER", v)
|
||||
if Enabled() {
|
||||
t.Fatalf("Enabled() true for RSMON_TEST_DOCKER=%q", v)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestFixtureDefaults(t *testing.T) {
|
||||
fixtures := Fixtures()
|
||||
if len(fixtures) != 3 {
|
||||
t.Fatalf("Fixtures() = %d, want 3", len(fixtures))
|
||||
}
|
||||
byName := map[string]Fixture{}
|
||||
for _, f := range fixtures {
|
||||
byName[f.Name] = f
|
||||
}
|
||||
if f := byName["alpine"]; f.ImageRef() != "reg.rsxx.ru/library/alpine:3" || f.Distro != "alpine" {
|
||||
t.Fatalf("alpine fixture wrong: %+v", f)
|
||||
}
|
||||
if f := byName["ubuntu"]; f.ImageRef() != "ubuntu:24.04" || f.Distro != "ubuntu" {
|
||||
t.Fatalf("ubuntu fixture wrong: %+v", f)
|
||||
}
|
||||
if f := byName["arch"]; f.ImageRef() != "archlinux:latest" || f.Distro != "arch" {
|
||||
t.Fatalf("arch fixture wrong: %+v", f)
|
||||
}
|
||||
if f := byName["alpine"]; f.UserOrDefault() != "root" {
|
||||
t.Fatalf("default user = %q, want root", f.UserOrDefault())
|
||||
}
|
||||
}
|
||||
|
||||
func TestFixtureImageEnvOverride(t *testing.T) {
|
||||
f := Fixture{Name: "ubuntu"}
|
||||
t.Setenv("RSMON_TEST_IMAGE_UBUNTU", "reg.rsxx.ru/library/ubuntu:24.04")
|
||||
if got := f.ImageRef(); got != "reg.rsxx.ru/library/ubuntu:24.04" {
|
||||
t.Fatalf("env override not applied: %q", got)
|
||||
}
|
||||
t.Setenv("RSMON_TEST_IMAGE_UBUNTU", "")
|
||||
if got := f.ImageRef(); got != "ubuntu:24.04" {
|
||||
t.Fatalf("default image changed after env cleared: %q", got)
|
||||
}
|
||||
|
||||
override := Fixture{Name: "arch", Image: "archlinux:2026.01.01"}
|
||||
if got := override.ImageRef(); got != "archlinux:2026.01.01" {
|
||||
t.Fatalf("fixture image override not applied: %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFixturesHaveDockerfiles(t *testing.T) {
|
||||
for _, f := range Fixtures() {
|
||||
path := f.DockerfilePath()
|
||||
if _, err := os.Stat(path); err != nil {
|
||||
t.Fatalf("fixture %s missing Dockerfile at %s: %v", f.Name, path, err)
|
||||
}
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.Contains(string(data), "FROM ") {
|
||||
t.Fatalf("fixture %s Dockerfile has no FROM", f.Name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewValidatesName(t *testing.T) {
|
||||
for _, bad := range []string{"", "with space", "UPPER", "semi;colon", "back`tick", strings.Repeat("a", 65)} {
|
||||
if _, err := New(bad, Fixtures()[0]); err == nil {
|
||||
t.Fatalf("New(%q) succeeded", bad)
|
||||
}
|
||||
}
|
||||
h, err := New("fixture-alpine", Fixtures()[0])
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if h.ContainerName() == "" || h.NetworkName() == "" {
|
||||
t.Fatalf("empty container/network names: %+v", h)
|
||||
}
|
||||
if h.Addr() != "" {
|
||||
t.Fatalf("Addr() = %q before Start, want empty", h.Addr())
|
||||
}
|
||||
}
|
||||
|
||||
func TestStartDisabled(t *testing.T) {
|
||||
t.Setenv("RSMON_TEST_DOCKER", "")
|
||||
h, err := New("unit-start", Fixtures()[0])
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := h.Start(context.Background()); err == nil {
|
||||
t.Fatal("Start succeeded with integration disabled")
|
||||
} else if !strings.Contains(err.Error(), "RSMON_TEST_DOCKER") {
|
||||
t.Fatalf("disabled Start error = %v, want opt-in hint", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDialBeforeStart(t *testing.T) {
|
||||
h, err := New("unit-dial", Fixtures()[0])
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := h.Dial(); err == nil {
|
||||
t.Fatal("Dial succeeded before Start")
|
||||
}
|
||||
}
|
||||
|
||||
func TestParsePublishedPort(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
in string
|
||||
want int
|
||||
}{
|
||||
{in: "127.0.0.1:49153\n", want: 49153},
|
||||
{in: "0.0.0.0:2222", want: 2222},
|
||||
{in: "127.0.0.1:0\n", want: 0},
|
||||
{in: "::1:32768\n", want: 32768},
|
||||
{in: "127.0.0.1:notaport\n", want: 0},
|
||||
{in: "", want: 0},
|
||||
} {
|
||||
got, err := parsePublishedPort(tc.in)
|
||||
if tc.want == 0 {
|
||||
if err == nil {
|
||||
t.Fatalf("parsePublishedPort(%q) succeeded with %d", tc.in, got)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if err != nil || got != tc.want {
|
||||
t.Fatalf("parsePublishedPort(%q) = %d, %v; want %d", tc.in, got, err, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestWaitForPort exercises the readiness probe against a real local
|
||||
// listener so the polling loop is covered without Docker.
|
||||
func TestWaitForPort(t *testing.T) {
|
||||
ln, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer ln.Close() //nolint:errcheck
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
if err := waitForPort(ctx, ln.Addr().String(), 2*time.Second); err != nil {
|
||||
t.Fatalf("waitForPort on open listener: %v", err)
|
||||
}
|
||||
|
||||
// A port that never opens must time out.
|
||||
closed, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
addr := closed.Addr().String()
|
||||
closed.Close() //nolint:errcheck
|
||||
|
||||
short, shortCancel := context.WithTimeout(context.Background(), 3*time.Second)
|
||||
defer shortCancel()
|
||||
if err := waitForPort(short, addr, 1500*time.Millisecond); err == nil {
|
||||
t.Fatal("waitForPort on closed port succeeded")
|
||||
}
|
||||
}
|
||||
|
||||
func TestWriteKnownHostsRoundTrip(t *testing.T) {
|
||||
// Parsing a known_hosts entry requires a real key; use the bundled
|
||||
// public key so the format is exercised.
|
||||
raw, err := os.ReadFile(filepath.Join("testdata", "keys", "rsmon_test_ed25519.pub"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
key, _, _, _, err := ssh.ParseAuthorizedKey(raw)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
path, err := writeKnownHosts(t.TempDir(), "127.0.0.1:49153", key)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.Contains(string(data), "[127.0.0.1]:49153") {
|
||||
t.Fatalf("known_hosts entry %q missing bracketed address", data)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateName(t *testing.T) {
|
||||
for _, ok := range []string{"alpine", "fixture-1", "a", strings.Repeat("x", 64)} {
|
||||
if err := validateName(ok); err != nil {
|
||||
t.Fatalf("validateName(%q): %v", ok, err)
|
||||
}
|
||||
}
|
||||
for _, bad := range []string{"", "A", "a b", "a/b", strings.Repeat("x", 65)} {
|
||||
if err := validateName(bad); err == nil {
|
||||
t.Fatalf("validateName(%q) succeeded", bad)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSetDockerBin(t *testing.T) {
|
||||
SetDockerBin("docker")
|
||||
SetDockerBin("")
|
||||
if dockerBin != "docker" {
|
||||
t.Fatalf("dockerBin = %q after reset", dockerBin)
|
||||
}
|
||||
SetDockerBin("/stub/docker")
|
||||
if dockerBin != "/stub/docker" {
|
||||
t.Fatalf("dockerBin = %q after set", dockerBin)
|
||||
}
|
||||
SetDockerBin("")
|
||||
}
|
||||
|
||||
func TestDockerDNSOverride(t *testing.T) {
|
||||
t.Setenv("RSMON_TEST_DOCKER_DNS", "")
|
||||
if got := dockerDNS(); len(got) != 0 {
|
||||
t.Fatalf("dockerDNS() with empty env = %v, want none", got)
|
||||
}
|
||||
t.Setenv("RSMON_TEST_DOCKER_DNS", "8.8.8.8, 1.1.1.1")
|
||||
if got := dockerDNS(); len(got) != 4 || got[0] != "--dns" || got[1] != "8.8.8.8" || got[3] != "1.1.1.1" {
|
||||
t.Fatalf("dockerDNS() = %v", got)
|
||||
}
|
||||
t.Setenv("RSMON_TEST_DOCKER_DNS", " ,,")
|
||||
if got := dockerDNS(); len(got) != 0 {
|
||||
t.Fatalf("dockerDNS() with blank entries = %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// writeStubDocker installs a fake docker binary that records its argv to
|
||||
// logPath and returns the recorded path. The stub succeeds for build,
|
||||
// network, run, and teardown calls; `port` fails so Start fails after
|
||||
// the container and network exist. Setting STUB_DOCKER_FAIL_BUILD=1 makes
|
||||
// the build step fail instead. Real Docker is never touched.
|
||||
func writeStubDocker(t *testing.T, logPath string) string {
|
||||
t.Helper()
|
||||
dir := t.TempDir()
|
||||
stub := filepath.Join(dir, "docker")
|
||||
script := `#!/bin/sh
|
||||
echo "$*" >> "$STUB_DOCKER_LOG"
|
||||
case "$1" in
|
||||
build)
|
||||
if [ "${STUB_DOCKER_FAIL_BUILD:-0}" = "1" ]; then
|
||||
echo "stub build failure" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "sha256:stub-image-id"
|
||||
;;
|
||||
network) echo "stub-network-id" ;;
|
||||
run) echo "stub-container-id" ;;
|
||||
port) echo "stub port failure" >&2; exit 1 ;;
|
||||
rm|-r|image|ps) exit 0 ;;
|
||||
*) exit 0 ;;
|
||||
esac
|
||||
`
|
||||
if err := os.WriteFile(stub, []byte(script), 0o700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Setenv("STUB_DOCKER_LOG", logPath)
|
||||
SetDockerBin(stub)
|
||||
t.Cleanup(func() { SetDockerBin("") })
|
||||
return stub
|
||||
}
|
||||
|
||||
func readDockerCalls(t *testing.T, logPath string) string {
|
||||
t.Helper()
|
||||
calls, err := os.ReadFile(logPath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return string(calls)
|
||||
}
|
||||
|
||||
// TestStartFailureCleansUpResources drives Start to failure after the
|
||||
// container and network were created (the stub `docker port` fails) and
|
||||
// asserts the failure path removed every resource: container, network,
|
||||
// fixture image, and no temp known_hosts dir leaked.
|
||||
func TestStartFailureCleansUpResources(t *testing.T) {
|
||||
t.Setenv("RSMON_TEST_DOCKER", "1")
|
||||
logPath := filepath.Join(t.TempDir(), "docker-calls.log")
|
||||
writeStubDocker(t, logPath)
|
||||
|
||||
h, err := New("fail-cleanup", Fixtures()[0])
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := h.Start(context.Background()); err == nil {
|
||||
t.Fatal("Start succeeded despite stub docker port failure")
|
||||
}
|
||||
calls := readDockerCalls(t, logPath)
|
||||
for _, want := range []string{"build", "network create", "run -d", "rm -f", "network rm", "image rm"} {
|
||||
if !strings.Contains(calls, want) {
|
||||
t.Fatalf("failed-start cleanup missing docker call %q; calls:\n%s", want, calls)
|
||||
}
|
||||
}
|
||||
if h.KnownHostsPath() != "" {
|
||||
t.Fatalf("known_hosts path set after failed start: %q", h.KnownHostsPath())
|
||||
}
|
||||
}
|
||||
|
||||
// TestStartBuildFailureIsSafe drives Start to fail at the very first
|
||||
// step (build) and asserts teardown stays idempotent and harmless: no
|
||||
// container or network was ever created, and the failure path leaves
|
||||
// nothing behind.
|
||||
func TestStartBuildFailureIsSafe(t *testing.T) {
|
||||
t.Setenv("RSMON_TEST_DOCKER", "1")
|
||||
t.Setenv("STUB_DOCKER_FAIL_BUILD", "1")
|
||||
logPath := filepath.Join(t.TempDir(), "docker-calls.log")
|
||||
writeStubDocker(t, logPath)
|
||||
|
||||
h, err := New("fail-build", Fixtures()[0])
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := h.Start(context.Background()); err == nil {
|
||||
t.Fatal("Start succeeded despite stub docker build failure")
|
||||
}
|
||||
if h.ContainerID() != "" {
|
||||
t.Fatalf("container id set after build failure: %q", h.ContainerID())
|
||||
}
|
||||
if h.KnownHostsPath() != "" {
|
||||
t.Fatalf("known_hosts path set after build failure: %q", h.KnownHostsPath())
|
||||
}
|
||||
if err := h.Stop(); err != nil {
|
||||
t.Fatalf("Stop after failed build: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestStopRemovesKnownHostsDir verifies Stop deletes the temp known_hosts
|
||||
// directory even when no container was ever started (the failed-start
|
||||
// path registers the dir before the final write).
|
||||
func TestStopRemovesKnownHostsDir(t *testing.T) {
|
||||
logPath := filepath.Join(t.TempDir(), "docker-calls.log")
|
||||
writeStubDocker(t, logPath)
|
||||
|
||||
h, err := New("stop-dir", Fixtures()[0])
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
dir := t.TempDir()
|
||||
h.knownHosts = filepath.Join(dir, "known_hosts")
|
||||
if err := os.WriteFile(h.knownHosts, []byte("placeholder"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := h.Stop(); err != nil {
|
||||
t.Fatalf("Stop: %v", err)
|
||||
}
|
||||
if _, err := os.Stat(dir); !os.IsNotExist(err) {
|
||||
t.Fatalf("known_hosts temp dir still exists after Stop: %v", err)
|
||||
}
|
||||
if h.KnownHostsPath() != "" {
|
||||
t.Fatalf("KnownHostsPath = %q after Stop, want empty", h.KnownHostsPath())
|
||||
}
|
||||
}
|
||||
|
||||
// TestStopIdempotent verifies repeated Stop calls do not error.
|
||||
func TestStopIdempotent(t *testing.T) {
|
||||
logPath := filepath.Join(t.TempDir(), "docker-calls.log")
|
||||
writeStubDocker(t, logPath)
|
||||
|
||||
h, err := New("stop-again", Fixtures()[0])
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
h.container = "rsmon-worker-test-none"
|
||||
h.network = "rsmon-worker-test-none"
|
||||
h.imageTag = "rsmon-worker-test/none:local"
|
||||
h.knownHosts = filepath.Join(t.TempDir(), "known_hosts")
|
||||
if err := h.Stop(); err != nil {
|
||||
t.Fatalf("first Stop: %v", err)
|
||||
}
|
||||
if err := h.Stop(); err != nil {
|
||||
t.Fatalf("second Stop: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestImageTagUnique verifies every harness instance gets its own image
|
||||
// tag so teardown can never delete another instance's image.
|
||||
func TestImageTagUnique(t *testing.T) {
|
||||
a, err := New("img-a", Fixtures()[0])
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b, err := New("img-b", Fixtures()[0])
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if a.ImageTag() == b.ImageTag() {
|
||||
t.Fatalf("image tags collide: %q", a.ImageTag())
|
||||
}
|
||||
if !strings.HasPrefix(a.ImageTag(), "rsmon-worker-test/alpine-") {
|
||||
t.Fatalf("unexpected image tag: %q", a.ImageTag())
|
||||
}
|
||||
}
|
||||
393
internal/installer/harness/integration_activation_test.go
Обычный файл
393
internal/installer/harness/integration_activation_test.go
Обычный файл
@@ -0,0 +1,393 @@
|
||||
package harness
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"golang.org/x/crypto/ssh"
|
||||
|
||||
"rocketgit.ru/rsmon/worker/internal/installer"
|
||||
"rocketgit.ru/rsmon/worker/internal/sshinstall"
|
||||
)
|
||||
|
||||
// testWorkerToken is the operator-supplied worker token used by the
|
||||
// activation E2E tests. The fixture has no reachable control plane, but
|
||||
// the worker still binds its webapp and serves /healthz, which is what
|
||||
// activation verifies.
|
||||
const testWorkerToken = "e2e-activation-test-token"
|
||||
|
||||
// activationOptions builds the work-package-4 source-install options for
|
||||
// a started fixture: full activation with a fixed test token, honoring
|
||||
// the RSMON_TEST_SOURCE_REPO/BRANCH overrides the staging tests use.
|
||||
func activationOptions(h *Harness, f Fixture) installer.SourceInstallOptions {
|
||||
opts := installer.SourceInstallOptions{
|
||||
SSHOptions: installer.SSHOptions{
|
||||
Host: "127.0.0.1",
|
||||
Port: h.Port(),
|
||||
User: f.UserOrDefault(),
|
||||
IdentityFile: testKeyPath(),
|
||||
KnownHostsFile: h.KnownHostsPath(),
|
||||
},
|
||||
Activation: installer.ActivationOptions{
|
||||
Activate: true,
|
||||
URL: "https://rsmon.ru",
|
||||
Token: testWorkerToken,
|
||||
},
|
||||
}
|
||||
if repo := strings.TrimSpace(os.Getenv("RSMON_TEST_SOURCE_REPO")); repo != "" {
|
||||
opts.Repo = repo
|
||||
}
|
||||
if branch := strings.TrimSpace(os.Getenv("RSMON_TEST_SOURCE_BRANCH")); branch != "" {
|
||||
opts.Branch = branch
|
||||
}
|
||||
return opts
|
||||
}
|
||||
|
||||
// fixtureByName returns the fixture entry for a fixture name.
|
||||
func fixtureByName(t *testing.T, name string) Fixture {
|
||||
t.Helper()
|
||||
for _, f := range Fixtures() {
|
||||
if f.Name == name {
|
||||
return f
|
||||
}
|
||||
}
|
||||
t.Fatalf("no fixture named %q", name)
|
||||
return Fixture{}
|
||||
}
|
||||
|
||||
// startActivatedFixture starts a fixture container and runs a full
|
||||
// source install with activation against it, returning the harness, the
|
||||
// options (so a test can rerun with modifications), the result, and a
|
||||
// live SSH client. The caller owns client.Close and h.Stop (registered as
|
||||
// a test cleanup).
|
||||
func startActivatedFixture(t *testing.T, name string) (*Harness, installer.SourceInstallOptions, *installer.SourceInstallResult, *ssh.Client) {
|
||||
t.Helper()
|
||||
f := fixtureByName(t, name)
|
||||
h, err := New("activate-"+name, f)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute)
|
||||
defer cancel()
|
||||
if err := h.Start(ctx); err != nil {
|
||||
t.Fatalf("start %s fixture: %v", name, err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := h.Stop(); err != nil {
|
||||
t.Errorf("stop %s fixture: %v", name, err)
|
||||
}
|
||||
})
|
||||
opts := activationOptions(h, f)
|
||||
res, err := installer.SourceInstall(opts)
|
||||
if err != nil {
|
||||
t.Fatalf("activated source install on %s: %v", name, err)
|
||||
}
|
||||
client, err := h.Dial()
|
||||
if err != nil {
|
||||
t.Fatalf("dial %s after install: %v", name, err)
|
||||
}
|
||||
return h, opts, res, client
|
||||
}
|
||||
|
||||
// countWorkerProcesses returns the number of running processes whose
|
||||
// command line is exactly the installed worker binary. The anchored -f
|
||||
// pattern matches both busybox and procps pgrep (busybox's -x matches the
|
||||
// full argv, so -x cannot be used portably).
|
||||
func countWorkerProcesses(t *testing.T, client *ssh.Client, binary string) string {
|
||||
t.Helper()
|
||||
out, err := RunCommand(client, "pgrep -f '^"+binary+"$' | wc -l")
|
||||
if err != nil {
|
||||
t.Fatalf("count worker processes: %v", err)
|
||||
}
|
||||
return strings.TrimSpace(string(out))
|
||||
}
|
||||
|
||||
// assertActivatedService verifies the full work-package-4 remote state:
|
||||
// the installed layout and permissions, the native service definition for
|
||||
// the detected init, exactly one running worker process, the pid file,
|
||||
// and a live /healthz (via the installed binary's own liveness
|
||||
// subcommand).
|
||||
func assertActivatedService(t *testing.T, client *ssh.Client, name string, res *installer.SourceInstallResult) {
|
||||
t.Helper()
|
||||
a := res.Activation
|
||||
if a == nil {
|
||||
t.Fatalf("%s: activation result missing", name)
|
||||
}
|
||||
|
||||
out, err := RunCommand(client, "test -x "+shellQuote(a.Binary)+" && echo BIN_OK")
|
||||
if err != nil || !strings.Contains(string(out), "BIN_OK") {
|
||||
t.Fatalf("%s: installed binary not present/executable at %s: %q, %v", name, a.Binary, out, err)
|
||||
}
|
||||
|
||||
envMode, err := RunCommand(client, "stat -c '%a' "+shellQuote(a.EnvFile))
|
||||
if err != nil || strings.TrimSpace(string(envMode)) != "600" {
|
||||
t.Fatalf("%s: env file mode = %q, want 600 (%v)", name, envMode, err)
|
||||
}
|
||||
cfgMode, err := RunCommand(client, "stat -c '%a' "+shellQuote(a.ConfigDir))
|
||||
if err != nil || strings.TrimSpace(string(cfgMode)) != "750" {
|
||||
t.Fatalf("%s: config dir mode = %q, want 750 (%v)", name, cfgMode, err)
|
||||
}
|
||||
if _, err := RunCommand(client, "test -d "+shellQuote(a.DataDir)+"/webapp && echo DATA_OK"); err != nil {
|
||||
t.Fatalf("%s: data dir not created at %s: %v", name, a.DataDir, err)
|
||||
}
|
||||
|
||||
// The fixtures run no init system (sshd is PID 1), so activation
|
||||
// must use the embedded supervisor; the native service definition is
|
||||
// still installed for real hosts.
|
||||
if a.Supervisor != "none" {
|
||||
t.Fatalf("%s: supervisor = %q, want none in the fixture", name, a.Supervisor)
|
||||
}
|
||||
unitMode := "644"
|
||||
if res.Detection.InitSystem == sshinstall.InitOpenRC {
|
||||
unitMode = "755"
|
||||
}
|
||||
if a.UnitFile == "" {
|
||||
t.Fatalf("%s: no service definition installed", name)
|
||||
}
|
||||
if _, err := RunCommand(client, "test -f "+shellQuote(a.UnitFile)+" && echo UNIT_OK"); err != nil {
|
||||
t.Fatalf("%s: unit file missing at %s: %v", name, a.UnitFile, err)
|
||||
}
|
||||
gotUnitMode, err := RunCommand(client, "stat -c '%a' "+shellQuote(a.UnitFile))
|
||||
if err != nil || strings.TrimSpace(string(gotUnitMode)) != unitMode {
|
||||
t.Fatalf("%s: unit mode = %q, want %s (%v)", name, gotUnitMode, unitMode, err)
|
||||
}
|
||||
|
||||
// Exactly one worker process and a live /healthz.
|
||||
if got := countWorkerProcesses(t, client, a.Binary); got != "1" {
|
||||
t.Fatalf("%s: worker process count = %q, want 1", name, got)
|
||||
}
|
||||
if out, err := RunCommand(client, shellQuote(a.Binary)+" liveness"); err != nil || !strings.Contains(string(out), "ok") {
|
||||
t.Fatalf("%s: installed worker /healthz check failed: %q, %v", name, out, err)
|
||||
}
|
||||
pidOut, err := RunCommand(client, "cat "+shellQuote(a.DataDir)+"/worker.pid")
|
||||
if err != nil {
|
||||
t.Fatalf("%s: pid file missing: %v", name, err)
|
||||
}
|
||||
if _, err := RunCommand(client, "kill -0 "+strings.TrimSpace(string(pidOut))+" && echo PID_OK"); err != nil {
|
||||
t.Fatalf("%s: pid %s not alive: %v", name, pidOut, err)
|
||||
}
|
||||
}
|
||||
|
||||
// assertNoActivationLeaks verifies a rerun left no backup dir, no lock,
|
||||
// no /tmp upload leftovers, and no extra worker processes.
|
||||
func assertNoActivationLeaks(t *testing.T, client *ssh.Client, name string, dataDir string) {
|
||||
t.Helper()
|
||||
out, err := RunCommand(client,
|
||||
"leak=0; [ -e "+shellQuote(dataDir)+"/.rsmon-backup ] && leak=1; "+
|
||||
"[ -d "+shellQuote(dataDir)+"/.rsmon-activate.lock ] && leak=1; "+
|
||||
"ls /tmp 2>/dev/null | grep -q '^rsmon-worker-act\\.' && leak=1; "+
|
||||
"[ \"$leak\" -eq 1 ] && echo LEAK || echo CLEAN")
|
||||
if err != nil || strings.TrimSpace(string(out)) != "CLEAN" {
|
||||
t.Fatalf("%s: activation leftovers after rerun: %q, %v", name, out, err)
|
||||
}
|
||||
}
|
||||
|
||||
// installedBinarySHA captures the SHA-256 of the installed worker binary
|
||||
// so failure tests can prove the prior binary is preserved.
|
||||
func installedBinarySHA(t *testing.T, client *ssh.Client, a *installer.ActivationResult) []byte {
|
||||
t.Helper()
|
||||
out, err := RunCommand(client, "sha256sum "+shellQuote(a.Binary))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return bytes.TrimSpace(out)
|
||||
}
|
||||
|
||||
// TestSourceInstallActivationFixtures is the work-package-4 success and
|
||||
// idempotency acceptance test: on every fixture a full source install
|
||||
// ends with an atomically installed binary, a mode-0600 env file, the
|
||||
// data dir, a service definition for the detected init, and a running
|
||||
// verified worker. A rerun succeeds, keeps a single worker process,
|
||||
// rewrites the env deterministically, and leaks no temp or backup files.
|
||||
//
|
||||
// Opt-in: set RSMON_TEST_DOCKER=1 (see make test-ssh).
|
||||
func TestSourceInstallActivationFixtures(t *testing.T) {
|
||||
SkipUnlessEnabled(t)
|
||||
|
||||
for _, f := range Fixtures() {
|
||||
f := f
|
||||
t.Run(f.Name, func(t *testing.T) {
|
||||
_, opts, res, client := startActivatedFixture(t, f.Name)
|
||||
defer client.Close() //nolint:errcheck
|
||||
assertActivatedService(t, client, f.Name, res)
|
||||
|
||||
// Rerun: idempotent, single service, no leaks.
|
||||
res2, err := installer.SourceInstall(opts)
|
||||
if err != nil {
|
||||
t.Fatalf("%s: activated rerun: %v", f.Name, err)
|
||||
}
|
||||
if res2.Activation == nil || res2.ResolvedCommit != res.ResolvedCommit {
|
||||
t.Fatalf("%s: rerun lost activation/commit: %+v", f.Name, res2)
|
||||
}
|
||||
assertActivatedService(t, client, f.Name, res2)
|
||||
assertNoActivationLeaks(t, client, f.Name, res2.Activation.DataDir)
|
||||
|
||||
envAfter, err := RunCommand(client, "cat "+shellQuote(res2.Activation.EnvFile))
|
||||
if err != nil {
|
||||
t.Fatalf("%s: read env after rerun: %v", f.Name, err)
|
||||
}
|
||||
if !strings.Contains(string(envAfter), "RSMON_TOKEN="+testWorkerToken) {
|
||||
t.Fatalf("%s: env lost the worker token after rerun: %q", f.Name, envAfter)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestSourceInstallActivationFailureRollback is the work-package-4
|
||||
// rollback acceptance test. After a successful activated install on the
|
||||
// Alpine fixture, each rerun is forced to fail at a different stage -
|
||||
// build (dirty checkout), activation (atomic swap sabotaged), start (the
|
||||
// worker process dies at boot), and health (/healthz unreachable) - and
|
||||
// every failure must leave the previous working install running with the
|
||||
// same binary, the same single process, and /healthz answering.
|
||||
//
|
||||
// Opt-in: set RSMON_TEST_DOCKER=1 (see make test-ssh).
|
||||
func TestSourceInstallActivationFailureRollback(t *testing.T) {
|
||||
SkipUnlessEnabled(t)
|
||||
|
||||
const fixture = "alpine"
|
||||
_, opts, res, client := startActivatedFixture(t, fixture)
|
||||
defer client.Close() //nolint:errcheck
|
||||
a := res.Activation
|
||||
assertActivatedService(t, client, fixture, res)
|
||||
|
||||
// 1. Build failure preserves the prior install.
|
||||
// A dirty tracked tree makes the rerun's checkout refuse before
|
||||
// the build, so neither staging nor the service changes. Appending
|
||||
// a marker is deterministic regardless of the cloned history (a
|
||||
// `git checkout master~1 -- <file>` would depend on whether that
|
||||
// commit differs from the resolved one).
|
||||
if _, err := RunCommand(client, "printf '\\n# rsmon-worker dirty-tree marker\\n' >> "+shellQuote(res.Plan.BuildDir)+"/Makefile"); err != nil {
|
||||
t.Fatalf("dirty the working tree: %v", err)
|
||||
}
|
||||
before := installedBinarySHA(t, client, a)
|
||||
_, err := installer.SourceInstall(opts)
|
||||
if err == nil || !strings.Contains(err.Error(), "check out branch") {
|
||||
t.Fatalf("rerun on dirty tree err = %v, want checkout failure", err)
|
||||
}
|
||||
assertActivatedService(t, client, fixture, res)
|
||||
if !bytes.Equal(before, installedBinarySHA(t, client, a)) {
|
||||
t.Fatalf("installed binary changed after build-failure rerun")
|
||||
}
|
||||
// Restore the tree so later reruns can check out again.
|
||||
if _, err := RunCommand(client, "git -C "+shellQuote(res.Plan.BuildDir)+" checkout -q HEAD -- Makefile"); err != nil {
|
||||
t.Fatalf("restore working tree: %v", err)
|
||||
}
|
||||
|
||||
// 2. Activation failure preserves the prior install.
|
||||
// Pre-creating the atomic-swap temp path as a directory makes the
|
||||
// binary install step fail mid-activation; the rollback trap must
|
||||
// restore the previous binary/env and restart the prior service.
|
||||
if _, err := RunCommand(client, "mkdir -p /usr/local/bin/rsmon-worker.new"); err != nil {
|
||||
t.Fatalf("sabotage the atomic binary swap: %v", err)
|
||||
}
|
||||
_, err = installer.SourceInstall(opts)
|
||||
if err == nil || !strings.Contains(err.Error(), "activate worker service") {
|
||||
t.Fatalf("activation-failure rerun err = %v, want activate failure", err)
|
||||
}
|
||||
assertActivatedService(t, client, fixture, res)
|
||||
if !bytes.Equal(before, installedBinarySHA(t, client, a)) {
|
||||
t.Fatalf("installed binary changed after activation-failure rerun")
|
||||
}
|
||||
assertNoActivationLeaks(t, client, fixture, a.DataDir)
|
||||
if _, err := RunCommand(client, "rm -rf /usr/local/bin/rsmon-worker.new"); err != nil {
|
||||
t.Fatalf("clean the swap sabotage: %v", err)
|
||||
}
|
||||
|
||||
// 3. Start failure preserves the prior install.
|
||||
// WORKER_CLUSTER_ENABLED=true without WORKER_LOGIN/PASSWORD makes
|
||||
// the new worker exit at boot (cluster init is fatal), so the
|
||||
// activation supervisor detects the process died and rolls back.
|
||||
envPath := filepath.Join(t.TempDir(), "worker.env")
|
||||
startEnv := "RSMON_URL=https://rsmon.ru\nRSMON_TOKEN=" + testWorkerToken + "\nWORKER_CLUSTER_ENABLED=true\n"
|
||||
if err := os.WriteFile(envPath, []byte(startEnv), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
startOpts := opts
|
||||
startOpts.Activation.EnvFile = envPath
|
||||
_, err = installer.SourceInstall(startOpts)
|
||||
if err == nil || !strings.Contains(err.Error(), "start failure") {
|
||||
t.Fatalf("start-failure rerun err = %v, want start failure", err)
|
||||
}
|
||||
assertActivatedService(t, client, fixture, res)
|
||||
if !bytes.Equal(before, installedBinarySHA(t, client, a)) {
|
||||
t.Fatalf("installed binary changed after start-failure rerun")
|
||||
}
|
||||
assertNoActivationLeaks(t, client, fixture, a.DataDir)
|
||||
|
||||
// 4. Health failure preserves the prior install.
|
||||
// WORKER_HOST=255.255.255.255 makes the new worker bind fail and
|
||||
// /healthz unreachable; the process stays up but the health gate
|
||||
// fails and the rollback restores the prior loopback deployment.
|
||||
healthOpts := opts
|
||||
healthOpts.Activation.Host = "255.255.255.255"
|
||||
_, err = installer.SourceInstall(healthOpts)
|
||||
if err == nil || !strings.Contains(err.Error(), "health failure") {
|
||||
t.Fatalf("health-failure rerun err = %v, want health failure", err)
|
||||
}
|
||||
assertActivatedService(t, client, fixture, res)
|
||||
if !bytes.Equal(before, installedBinarySHA(t, client, a)) {
|
||||
t.Fatalf("installed binary changed after health-failure rerun")
|
||||
}
|
||||
assertNoActivationLeaks(t, client, fixture, a.DataDir)
|
||||
}
|
||||
|
||||
// TestSourceInstallActivationNoStart installs the full layout (binary,
|
||||
// env, data dir, service definition) without starting the worker, and
|
||||
// asserts nothing runs afterward. Reruns stay idempotent.
|
||||
//
|
||||
// Opt-in: set RSMON_TEST_DOCKER=1 (see make test-ssh).
|
||||
func TestSourceInstallActivationNoStart(t *testing.T) {
|
||||
SkipUnlessEnabled(t)
|
||||
|
||||
const fixture = "alpine"
|
||||
f := fixtureByName(t, fixture)
|
||||
h, err := New("nostart-"+fixture, f)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute)
|
||||
defer cancel()
|
||||
if err := h.Start(ctx); err != nil {
|
||||
t.Fatalf("start %s fixture: %v", fixture, err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := h.Stop(); err != nil {
|
||||
t.Errorf("stop: %v", err)
|
||||
}
|
||||
})
|
||||
opts := activationOptions(h, f)
|
||||
opts.Activation.NoStart = true
|
||||
res, err := installer.SourceInstall(opts)
|
||||
if err != nil {
|
||||
t.Fatalf("no-start install: %v", err)
|
||||
}
|
||||
client, err := h.Dial()
|
||||
if err != nil {
|
||||
t.Fatalf("dial: %v", err)
|
||||
}
|
||||
defer client.Close() //nolint:errcheck
|
||||
a := res.Activation
|
||||
if a == nil || a.Started {
|
||||
t.Fatalf("no-start activation result = %+v", a)
|
||||
}
|
||||
if out, err := RunCommand(client, "test -x "+shellQuote(a.Binary)+" && test -f "+shellQuote(a.EnvFile)+" && echo INSTALLED"); err != nil || !strings.Contains(string(out), "INSTALLED") {
|
||||
t.Fatalf("no-start did not install the layout: %q, %v", out, err)
|
||||
}
|
||||
if got := countWorkerProcesses(t, client, a.Binary); got != "0" {
|
||||
t.Fatalf("no-start left a running worker: %q", got)
|
||||
}
|
||||
// A rerun is idempotent and still starts nothing.
|
||||
if _, err := installer.SourceInstall(opts); err != nil {
|
||||
t.Fatalf("no-start rerun: %v", err)
|
||||
}
|
||||
if got := countWorkerProcesses(t, client, a.Binary); got != "0" {
|
||||
t.Fatalf("no-start rerun left a running worker: %q", got)
|
||||
}
|
||||
assertNoActivationLeaks(t, client, fixture, a.DataDir)
|
||||
}
|
||||
533
internal/installer/harness/integration_test.go
Обычный файл
533
internal/installer/harness/integration_test.go
Обычный файл
@@ -0,0 +1,533 @@
|
||||
package harness
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/ed25519"
|
||||
"crypto/rand"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"golang.org/x/crypto/ssh"
|
||||
"golang.org/x/crypto/ssh/knownhosts"
|
||||
|
||||
"rocketgit.ru/rsmon/worker/internal/installer"
|
||||
"rocketgit.ru/rsmon/worker/internal/sshinstall"
|
||||
)
|
||||
|
||||
// TestHarnessFixtures is the work-package-1 acceptance test: each distro
|
||||
// fixture starts a real OpenSSH container, the harness waits for real
|
||||
// network readiness, and the installer's Go SSH client connects, runs
|
||||
// commands, and tears the environment down.
|
||||
//
|
||||
// Opt-in: set RSMON_TEST_DOCKER=1 (see make test-ssh).
|
||||
func TestHarnessFixtures(t *testing.T) {
|
||||
SkipUnlessEnabled(t)
|
||||
|
||||
for _, f := range Fixtures() {
|
||||
f := f
|
||||
t.Run(f.Name, func(t *testing.T) {
|
||||
h, err := New("fixture-"+f.Name, f)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Minute)
|
||||
defer cancel()
|
||||
if err := h.Start(ctx); err != nil {
|
||||
t.Fatalf("start %s fixture: %v", f.Name, err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := h.Stop(); err != nil {
|
||||
t.Errorf("stop %s fixture: %v", f.Name, err)
|
||||
}
|
||||
})
|
||||
|
||||
client, err := h.Dial()
|
||||
if err != nil {
|
||||
t.Fatalf("dial %s fixture: %v", f.Name, err)
|
||||
}
|
||||
defer client.Close() //nolint:errcheck
|
||||
|
||||
// 1. SSH command execution is real: round-trip a nonce.
|
||||
nonce := fmt.Sprintf("RSMON_SSH_OK_%d", time.Now().UnixNano())
|
||||
out, err := RunCommand(client, "printf '%s' "+shellQuote(nonce))
|
||||
if err != nil {
|
||||
t.Fatalf("ssh round trip: %v", err)
|
||||
}
|
||||
if strings.TrimSpace(string(out)) != nonce {
|
||||
t.Fatalf("ssh round trip = %q, want %q", out, nonce)
|
||||
}
|
||||
|
||||
// 2. The clean target has no Go toolchain and no worker source.
|
||||
out, err = RunCommand(client, "command -v go || true; test ! -e /usr/local/go && echo NO_GO; test ! -e /opt/rsmon-worker-src && echo NO_SOURCE")
|
||||
if err != nil {
|
||||
t.Fatalf("clean-state probe: %v", err)
|
||||
}
|
||||
clean := string(out)
|
||||
if strings.Contains(clean, "/go") && !strings.Contains(clean, "NO_GO") {
|
||||
t.Fatalf("fixture unexpectedly has Go installed: %q", clean)
|
||||
}
|
||||
if !strings.Contains(clean, "NO_SOURCE") {
|
||||
t.Fatalf("fixture unexpectedly has worker source: %q", clean)
|
||||
}
|
||||
|
||||
// 3. Distro detection over the real session matches the fixture.
|
||||
out, err = RunCommand(client, "cat /etc/os-release")
|
||||
if err != nil {
|
||||
t.Fatalf("read os-release: %v", err)
|
||||
}
|
||||
d := sshinstall.Detect(string(out), makeProber(client))
|
||||
if d.Distro != f.Distro {
|
||||
t.Fatalf("detected distro = %q, want %q (%s)", d.Distro, f.Distro, d.Summarize())
|
||||
}
|
||||
if d.PackageManager != f.Pkg {
|
||||
t.Fatalf("detected package manager = %q, want %q", d.PackageManager, f.Pkg)
|
||||
}
|
||||
if d.InitSystem != f.Init {
|
||||
t.Fatalf("detected init = %q, want %q", d.InitSystem, f.Init)
|
||||
}
|
||||
t.Logf("%s: %s", f.Name, d.Summarize())
|
||||
|
||||
// 4. A full source plan resolves for the detected host,
|
||||
// including the pinned Go toolchain for its real arch.
|
||||
out, err = RunCommand(client, "uname -m")
|
||||
if err != nil {
|
||||
t.Fatalf("uname -m: %v", err)
|
||||
}
|
||||
goarch, err := sshinstall.GoArch(strings.TrimSpace(string(out)))
|
||||
if err != nil {
|
||||
t.Fatalf("GoArch(%q): %v", out, err)
|
||||
}
|
||||
plan, err := sshinstall.PlanSource(d, sshinstall.SourceOptions{UnameM: strings.TrimSpace(string(out))})
|
||||
if err != nil {
|
||||
t.Fatalf("PlanSource: %v", err)
|
||||
}
|
||||
if plan.Toolchain.Arch != "linux-"+goarch {
|
||||
t.Fatalf("plan toolchain %q does not match detected arch %q", plan.Toolchain.Arch, goarch)
|
||||
}
|
||||
if len(plan.Packages) == 0 || plan.Repo == "" {
|
||||
t.Fatalf("incomplete plan: %+v", plan)
|
||||
}
|
||||
steps := plan.Steps()
|
||||
if len(steps) != 6 {
|
||||
t.Fatalf("plan steps = %d, want 6", len(steps))
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestHarnessHostKeyMismatch verifies the security gate: a dial against
|
||||
// a known_hosts entry carrying a different host key must fail before any
|
||||
// command can run.
|
||||
func TestHarnessHostKeyMismatch(t *testing.T) {
|
||||
SkipUnlessEnabled(t)
|
||||
|
||||
f := Fixtures()[0] // alpine is the smallest fixture
|
||||
h, err := New("hostkey-"+f.Name, f)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Minute)
|
||||
defer cancel()
|
||||
if err := h.Start(ctx); err != nil {
|
||||
t.Fatalf("start fixture: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := h.Stop(); err != nil {
|
||||
t.Errorf("stop fixture: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
// Build a known_hosts entry with a different (freshly generated) key.
|
||||
_, priv, err := ed25519.GenerateKey(rand.Reader)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
signer, err := ssh.NewSignerFromKey(priv)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
wrongFile := filepath.Join(t.TempDir(), "known_hosts")
|
||||
line := knownhosts.Line([]string{h.Addr()}, signer.PublicKey())
|
||||
if err := os.WriteFile(wrongFile, []byte(line+"\n"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
keyBytes, err := os.ReadFile(testKeyPath())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
keySigner, err := ssh.ParsePrivateKey(keyBytes)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
callback, err := knownhosts.New(wrongFile)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
config := &ssh.ClientConfig{
|
||||
User: f.UserOrDefault(),
|
||||
Auth: []ssh.AuthMethod{ssh.PublicKeys(keySigner)},
|
||||
HostKeyCallback: callback,
|
||||
Timeout: 15 * time.Second,
|
||||
}
|
||||
client, err := ssh.Dial("tcp", h.Addr(), config)
|
||||
if err == nil {
|
||||
client.Close() //nolint:errcheck
|
||||
t.Fatal("dial with a mismatched host key succeeded")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "knownhosts") && !strings.Contains(err.Error(), "key") {
|
||||
t.Fatalf("host-key mismatch error = %v, want a key/host verification failure", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestHarnessTeardown verifies Stop reliably removes the container, the
|
||||
// dedicated network, the per-instance fixture image tag (never a shared
|
||||
// base image), and the temp known_hosts directory.
|
||||
func TestHarnessTeardown(t *testing.T) {
|
||||
SkipUnlessEnabled(t)
|
||||
|
||||
f := Fixtures()[0]
|
||||
h, err := New("teardown-"+f.Name, f)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Minute)
|
||||
defer cancel()
|
||||
if err := h.Start(ctx); err != nil {
|
||||
t.Fatalf("start fixture: %v", err)
|
||||
}
|
||||
if h.ContainerID() == "" {
|
||||
t.Fatal("container id empty after start")
|
||||
}
|
||||
if h.KnownHostsPath() == "" {
|
||||
t.Fatal("known_hosts not created after start")
|
||||
}
|
||||
knownHostsDir := filepath.Dir(h.KnownHostsPath())
|
||||
if err := h.Stop(); err != nil {
|
||||
t.Fatalf("stop: %v", err)
|
||||
}
|
||||
if _, err := h.DockerCmd(ctx, "inspect", h.ContainerName()); err == nil {
|
||||
t.Fatal("container still present after Stop")
|
||||
}
|
||||
if _, err := h.DockerCmd(ctx, "network", "inspect", h.NetworkName()); err == nil {
|
||||
t.Fatal("network still present after Stop")
|
||||
}
|
||||
if _, err := h.DockerCmd(ctx, "image", "inspect", h.ImageTag()); err == nil {
|
||||
t.Fatalf("fixture image tag %q still present after Stop", h.ImageTag())
|
||||
}
|
||||
if _, err := os.Stat(knownHostsDir); !os.IsNotExist(err) {
|
||||
t.Fatalf("known_hosts temp dir %q still present after Stop: %v", knownHostsDir, err)
|
||||
}
|
||||
if h.KnownHostsPath() != "" {
|
||||
t.Fatalf("KnownHostsPath = %q after Stop, want empty", h.KnownHostsPath())
|
||||
}
|
||||
// Stop is idempotent.
|
||||
if err := h.Stop(); err != nil {
|
||||
t.Fatalf("second stop: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestHarnessHostKeyStableAcrossDial ensures the host key captured at
|
||||
// readiness is the one verified on every later dial, so a successful
|
||||
// Dial is proof of verified, real SSH transport.
|
||||
func TestHarnessHostKeyStable(t *testing.T) {
|
||||
SkipUnlessEnabled(t)
|
||||
|
||||
f := Fixtures()[0]
|
||||
h, err := New("key-"+f.Name, f)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Minute)
|
||||
defer cancel()
|
||||
if err := h.Start(ctx); err != nil {
|
||||
t.Fatalf("start fixture: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := h.Stop(); err != nil {
|
||||
t.Errorf("stop fixture: %v", err)
|
||||
}
|
||||
})
|
||||
client, err := h.Dial()
|
||||
if err != nil {
|
||||
t.Fatalf("dial: %v", err)
|
||||
}
|
||||
client.Close() //nolint:errcheck
|
||||
client, err = h.Dial()
|
||||
if err != nil {
|
||||
t.Fatalf("second dial: %v", err)
|
||||
}
|
||||
defer client.Close() //nolint:errcheck
|
||||
if out, err := RunCommand(client, "echo VERIFIED"); err != nil || strings.TrimSpace(string(out)) != "VERIFIED" {
|
||||
t.Fatalf("verified session command = %q, %v", out, err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestSourceInstallFixtures is the work-package-3 acceptance test: each
|
||||
// distro fixture starts clean (no Go, no worker source) and the real
|
||||
// installer executes the full source flow over SSH - prerequisite
|
||||
// install, verified Go toolchain download/extraction, clone/update of the
|
||||
// public repository, resolved branch/commit record, and a build to a
|
||||
// staging path. The running service and its config are deliberately not
|
||||
// installed (that is work package 4). A rerun exercises idempotency.
|
||||
//
|
||||
// Opt-in: set RSMON_TEST_DOCKER=1 (see make test-ssh).
|
||||
func TestSourceInstallFixtures(t *testing.T) {
|
||||
SkipUnlessEnabled(t)
|
||||
|
||||
for _, f := range Fixtures() {
|
||||
f := f
|
||||
t.Run(f.Name, func(t *testing.T) {
|
||||
h, err := New("source-"+f.Name, f)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute)
|
||||
defer cancel()
|
||||
if err := h.Start(ctx); err != nil {
|
||||
t.Fatalf("start %s fixture: %v", f.Name, err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := h.Stop(); err != nil {
|
||||
t.Errorf("stop %s fixture: %v", f.Name, err)
|
||||
}
|
||||
})
|
||||
|
||||
// Clean state: no Go toolchain, no source, no leftover
|
||||
// toolchain temp dirs.
|
||||
client, err := h.Dial()
|
||||
if err != nil {
|
||||
t.Fatalf("dial %s fixture: %v", f.Name, err)
|
||||
}
|
||||
probe, err := RunCommand(client, "command -v go || true; test ! -e /usr/local/go && echo NO_GO; test ! -e /opt/rsmon-worker-src && echo NO_SOURCE; ls /tmp | grep -q rsmon-toolchain && echo LEAK; ls /usr/local 2>/dev/null | grep -qE 'go-staging|go-backup' && echo LEAK; echo DONE")
|
||||
if err != nil {
|
||||
t.Fatalf("clean-state probe: %v", err)
|
||||
}
|
||||
clean := string(probe)
|
||||
if !strings.Contains(clean, "NO_GO") || !strings.Contains(clean, "NO_SOURCE") {
|
||||
t.Fatalf("fixture is not clean: %q", clean)
|
||||
}
|
||||
if strings.Contains(clean, "LEAK") {
|
||||
t.Fatalf("fixture has leftover toolchain temp dirs: %q", clean)
|
||||
}
|
||||
client.Close() //nolint:errcheck
|
||||
|
||||
opts := installer.SourceInstallOptions{
|
||||
SSHOptions: installer.SSHOptions{
|
||||
Host: "127.0.0.1",
|
||||
Port: h.Port(),
|
||||
User: f.UserOrDefault(),
|
||||
IdentityFile: testKeyPath(),
|
||||
KnownHostsFile: h.KnownHostsPath(),
|
||||
},
|
||||
// Staging-only: service activation (work package 4) is
|
||||
// exercised by the dedicated activation tests.
|
||||
Activation: installer.ActivationOptions{Activate: false},
|
||||
}
|
||||
if repo := strings.TrimSpace(os.Getenv("RSMON_TEST_SOURCE_REPO")); repo != "" {
|
||||
opts.Repo = repo
|
||||
}
|
||||
if branch := strings.TrimSpace(os.Getenv("RSMON_TEST_SOURCE_BRANCH")); branch != "" {
|
||||
opts.Branch = branch
|
||||
}
|
||||
|
||||
res, err := installer.SourceInstall(opts)
|
||||
if err != nil {
|
||||
t.Fatalf("source install on %s: %v", f.Name, err)
|
||||
}
|
||||
t.Logf("%s: %s -> branch=%s commit=%s staged=%s", f.Name, res.Detection.Summarize(),
|
||||
res.ResolvedBranch, res.ResolvedCommit, res.StageBinary)
|
||||
|
||||
if res.Detection.Distro != f.Distro || res.Detection.PackageManager != f.Pkg || res.Detection.InitSystem != f.Init {
|
||||
t.Fatalf("detection = %+v, want %s/%s/%s", res.Detection, f.Distro, f.Pkg, f.Init)
|
||||
}
|
||||
if res.GoArch != "linux-"+strings.TrimPrefix(res.Plan.Toolchain.Arch, "linux-") {
|
||||
t.Fatalf("resolved arch = %q, want %q", res.GoArch, res.Plan.Toolchain.Arch)
|
||||
}
|
||||
if res.ResolvedBranch == "" || len(res.ResolvedCommit) != 40 {
|
||||
t.Fatalf("resolved branch/commit incomplete: %s @ %s", res.ResolvedBranch, res.ResolvedCommit)
|
||||
}
|
||||
if res.StageBinary == "" || res.RecordFile == "" || res.ToolchainDir == "" {
|
||||
t.Fatalf("result paths incomplete: %+v", res)
|
||||
}
|
||||
|
||||
client, err = h.Dial()
|
||||
if err != nil {
|
||||
t.Fatalf("redial: %v", err)
|
||||
}
|
||||
defer client.Close() //nolint:errcheck
|
||||
assertSourceInstallState(t, client, f.Name, res)
|
||||
|
||||
// Idempotent rerun: succeeds, resolves the same branch,
|
||||
// reuses the toolchain, and leaks no temp files.
|
||||
res2, err := installer.SourceInstall(opts)
|
||||
if err != nil {
|
||||
t.Fatalf("source install rerun on %s: %v", f.Name, err)
|
||||
}
|
||||
if res2.ResolvedBranch != res.ResolvedBranch || len(res2.ResolvedCommit) != 40 {
|
||||
t.Fatalf("rerun resolved = %s @ %s, want branch %s", res2.ResolvedBranch, res2.ResolvedCommit, res.ResolvedBranch)
|
||||
}
|
||||
assertSourceInstallState(t, client, f.Name, res2)
|
||||
if out, err := RunCommand(client, "leak=0; ls /tmp | grep -q rsmon-toolchain && leak=1; ls /usr/local 2>/dev/null | grep -qE 'go-staging|go-backup' && leak=1; [ \"$leak\" -eq 1 ] && echo LEAK || echo CLEAN"); err != nil || strings.TrimSpace(string(out)) != "CLEAN" {
|
||||
t.Fatalf("toolchain temp dirs leaked after rerun: %q, %v", out, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// assertSourceInstallState verifies the remote side-effects of a source
|
||||
// install: the toolchain reports the pinned version, the staging binary
|
||||
// exists and reports the resolved commit, and the record file carries
|
||||
// the resolved branch and commit.
|
||||
func assertSourceInstallState(t *testing.T, client *ssh.Client, name string, res *installer.SourceInstallResult) {
|
||||
t.Helper()
|
||||
|
||||
out, err := RunCommand(client, res.ToolchainDir+"/bin/go version")
|
||||
if err != nil {
|
||||
t.Fatalf("%s: toolchain missing: %v", name, err)
|
||||
}
|
||||
if !strings.Contains(string(out), "go"+res.Plan.Toolchain.Version) {
|
||||
t.Fatalf("%s: toolchain version = %q, want go%s", name, out, res.Plan.Toolchain.Version)
|
||||
}
|
||||
|
||||
out, err = RunCommand(client, "test -x "+shellQuote(res.StageBinary)+" && echo BUILT")
|
||||
if err != nil || !strings.Contains(string(out), "BUILT") {
|
||||
t.Fatalf("%s: staging binary not present at %s: %q, %v", name, res.StageBinary, out, err)
|
||||
}
|
||||
out, err = RunCommand(client, shellQuote(res.StageBinary)+" --version")
|
||||
if err != nil {
|
||||
t.Fatalf("%s: staging binary --version: %v", name, err)
|
||||
}
|
||||
if !strings.Contains(string(out), "commit="+res.ResolvedCommit[:12]) {
|
||||
t.Fatalf("%s: staging binary reports commit %q, want short %s", name, out, res.ResolvedCommit[:12])
|
||||
}
|
||||
|
||||
record, err := RunCommand(client, "cat "+shellQuote(res.RecordFile))
|
||||
if err != nil {
|
||||
t.Fatalf("%s: read record file: %v", name, err)
|
||||
}
|
||||
if !strings.Contains(string(record), "branch="+res.ResolvedBranch) || !strings.Contains(string(record), "commit="+res.ResolvedCommit) {
|
||||
t.Fatalf("%s: record file = %q, want branch=%s commit=%s", name, record, res.ResolvedBranch, res.ResolvedCommit)
|
||||
}
|
||||
}
|
||||
|
||||
// TestSourceInstallDirtyCheckoutPreservesStaging is the work-package-3
|
||||
// failure-atomicity test: after a successful install, dirtying the
|
||||
// tracked working tree makes the next checkout fail closed. The rerun
|
||||
// must report the checkout error without ever reaching the build step,
|
||||
// leaving the previous staging binary and commit record byte-for-byte
|
||||
// unchanged, and without leaking toolchain staging/backup directories.
|
||||
func TestSourceInstallDirtyCheckoutPreservesStaging(t *testing.T) {
|
||||
SkipUnlessEnabled(t)
|
||||
|
||||
f := Fixtures()[0] // alpine is the smallest fixture
|
||||
h, err := New("dirty-"+f.Name, f)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute)
|
||||
defer cancel()
|
||||
if err := h.Start(ctx); err != nil {
|
||||
t.Fatalf("start %s fixture: %v", f.Name, err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := h.Stop(); err != nil {
|
||||
t.Errorf("stop %s fixture: %v", f.Name, err)
|
||||
}
|
||||
})
|
||||
|
||||
opts := installer.SourceInstallOptions{
|
||||
SSHOptions: installer.SSHOptions{
|
||||
Host: "127.0.0.1",
|
||||
Port: h.Port(),
|
||||
User: f.UserOrDefault(),
|
||||
IdentityFile: testKeyPath(),
|
||||
KnownHostsFile: h.KnownHostsPath(),
|
||||
},
|
||||
Activation: installer.ActivationOptions{Activate: false},
|
||||
}
|
||||
if repo := strings.TrimSpace(os.Getenv("RSMON_TEST_SOURCE_REPO")); repo != "" {
|
||||
opts.Repo = repo
|
||||
}
|
||||
res, err := installer.SourceInstall(opts)
|
||||
if err != nil {
|
||||
t.Fatalf("initial source install: %v", err)
|
||||
}
|
||||
|
||||
client, err := h.Dial()
|
||||
if err != nil {
|
||||
t.Fatalf("dial: %v", err)
|
||||
}
|
||||
defer client.Close() //nolint:errcheck
|
||||
|
||||
// Dirty a tracked file so the rerun's checkout refuses to proceed.
|
||||
// Appending a marker is deterministic regardless of the cloned history
|
||||
// (a `git checkout master~1 -- <file>` would depend on whether that
|
||||
// commit differs from the resolved one), and an appended line to a
|
||||
// tracked file is always an unstaged modification `git diff --quiet`
|
||||
// catches.
|
||||
if _, err := RunCommand(client, "printf '\\n# rsmon-worker dirty-tree marker\\n' >> "+shellQuote(res.Plan.BuildDir)+"/Makefile"); err != nil {
|
||||
t.Fatalf("dirty the working tree: %v", err)
|
||||
}
|
||||
beforeBinary, err := RunCommand(client, "sha256sum "+shellQuote(res.StageBinary))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
beforeRecord, err := RunCommand(client, "cat "+shellQuote(res.RecordFile))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if _, err := installer.SourceInstall(opts); err == nil || !strings.Contains(err.Error(), "check out branch") {
|
||||
t.Fatalf("rerun err = %v, want checkout failure on dirty tree", err)
|
||||
}
|
||||
|
||||
afterBinary, err := RunCommand(client, "sha256sum "+shellQuote(res.StageBinary))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
afterRecord, err := RunCommand(client, "cat "+shellQuote(res.RecordFile))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !bytes.Equal(bytes.TrimSpace(beforeBinary), bytes.TrimSpace(afterBinary)) {
|
||||
t.Fatalf("staging binary changed after failed rerun:\nbefore: %s\nafter: %s", beforeBinary, afterBinary)
|
||||
}
|
||||
if !bytes.Equal(bytes.TrimSpace(beforeRecord), bytes.TrimSpace(afterRecord)) {
|
||||
t.Fatalf("commit record changed after failed rerun:\nbefore: %s\nafter: %s", beforeRecord, afterRecord)
|
||||
}
|
||||
if out, err := RunCommand(client, "leak=0; ls /tmp | grep -q rsmon-toolchain && leak=1; ls /usr/local 2>/dev/null | grep -qE 'go-staging|go-backup' && leak=1; [ \"$leak\" -eq 1 ] && echo LEAK || echo CLEAN"); err != nil || strings.TrimSpace(string(out)) != "CLEAN" {
|
||||
t.Fatalf("toolchain staging leaked after failed rerun: %q, %v", out, err)
|
||||
}
|
||||
}
|
||||
|
||||
// makeProber builds an sshinstall.FileProber over a live SSH session.
|
||||
func makeProber(client *ssh.Client) sshinstall.FileProber {
|
||||
return func(paths ...string) map[string]bool {
|
||||
// The trailing `; true` keeps the shell exit status 0: the
|
||||
// last `[ -e "$p" ]` in the loop would otherwise set exit 1
|
||||
// when the final path is absent (as on Arch), which is not an
|
||||
// error for a probe.
|
||||
expr := "for p in " + strings.Join(paths, " ") + "; do [ -e \"$p\" ] && printf '%s\\n' \"$p\"; done; true"
|
||||
out, err := RunCommand(client, expr)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
present := make(map[string]bool, len(paths))
|
||||
for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") {
|
||||
if line = strings.TrimSpace(line); line != "" {
|
||||
present[line] = true
|
||||
}
|
||||
}
|
||||
return present
|
||||
}
|
||||
}
|
||||
|
||||
func shellQuote(value string) string {
|
||||
return "'" + strings.ReplaceAll(value, "'", "'\\''") + "'"
|
||||
}
|
||||
21
internal/installer/harness/testdata/fixtures/alpine/Dockerfile
поставляемый
Обычный файл
21
internal/installer/harness/testdata/fixtures/alpine/Dockerfile
поставляемый
Обычный файл
@@ -0,0 +1,21 @@
|
||||
# Alpine OpenSSH fixture for the source-install harness.
|
||||
#
|
||||
# Base image is the reg.rsxx.ru mirror (alpine:3). openrc is installed
|
||||
# explicitly so init-system detection has a stable marker; openssh is the
|
||||
# minimal sshd. Password auth is disabled; the fixture authenticates with
|
||||
# the shared test key.
|
||||
FROM reg.rsxx.ru/library/alpine:3
|
||||
|
||||
RUN apk add --no-cache openssh openrc \
|
||||
&& mkdir -p /run/sshd /root/.ssh \
|
||||
&& chmod 700 /root/.ssh \
|
||||
&& ssh-keygen -A \
|
||||
&& sed -i 's/^#\?PermitRootLogin.*/PermitRootLogin prohibit-password/' /etc/ssh/sshd_config \
|
||||
&& sed -i 's/^#\?PasswordAuthentication.*/PasswordAuthentication no/' /etc/ssh/sshd_config \
|
||||
&& sed -i 's/^#\?PubkeyAuthentication.*/PubkeyAuthentication yes/' /etc/ssh/sshd_config
|
||||
|
||||
COPY keys/rsmon_test_ed25519.pub /root/.ssh/authorized_keys
|
||||
RUN chmod 600 /root/.ssh/authorized_keys
|
||||
|
||||
EXPOSE 22
|
||||
CMD ["/usr/sbin/sshd", "-D", "-e"]
|
||||
19
internal/installer/harness/testdata/fixtures/arch/Dockerfile
поставляемый
Обычный файл
19
internal/installer/harness/testdata/fixtures/arch/Dockerfile
поставляемый
Обычный файл
@@ -0,0 +1,19 @@
|
||||
# Arch Linux OpenSSH fixture for the source-install harness.
|
||||
#
|
||||
# No reg.rsxx.ru mirror exists for Arch yet, so the default is the
|
||||
# Docker Hub archlinux:latest image. Override with RSMON_TEST_IMAGE_ARCH.
|
||||
FROM archlinux:latest
|
||||
|
||||
RUN pacman -Sy --noconfirm --needed openssh \
|
||||
&& mkdir -p /run/sshd /root/.ssh \
|
||||
&& chmod 700 /root/.ssh \
|
||||
&& ssh-keygen -A \
|
||||
&& sed -i 's/^#\?PermitRootLogin.*/PermitRootLogin prohibit-password/' /etc/ssh/sshd_config \
|
||||
&& sed -i 's/^#\?PasswordAuthentication.*/PasswordAuthentication no/' /etc/ssh/sshd_config \
|
||||
&& sed -i 's/^#\?PubkeyAuthentication.*/PubkeyAuthentication yes/' /etc/ssh/sshd_config
|
||||
|
||||
COPY keys/rsmon_test_ed25519.pub /root/.ssh/authorized_keys
|
||||
RUN chmod 600 /root/.ssh/authorized_keys
|
||||
|
||||
EXPOSE 22
|
||||
CMD ["/usr/sbin/sshd", "-D", "-e"]
|
||||
20
internal/installer/harness/testdata/fixtures/ubuntu/Dockerfile
поставляемый
Обычный файл
20
internal/installer/harness/testdata/fixtures/ubuntu/Dockerfile
поставляемый
Обычный файл
@@ -0,0 +1,20 @@
|
||||
# Ubuntu OpenSSH fixture for the source-install harness.
|
||||
#
|
||||
# No reg.rsxx.ru mirror exists for Ubuntu yet, so the default is the
|
||||
# Docker Hub ubuntu:24.04 image. Override with RSMON_TEST_IMAGE_UBUNTU.
|
||||
FROM ubuntu:24.04
|
||||
|
||||
RUN apt-get update \
|
||||
&& DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends openssh-server \
|
||||
&& rm -rf /var/lib/apt/lists/* \
|
||||
&& mkdir -p /run/sshd /root/.ssh \
|
||||
&& chmod 700 /root/.ssh \
|
||||
&& sed -i 's/^#\?PermitRootLogin.*/PermitRootLogin prohibit-password/' /etc/ssh/sshd_config \
|
||||
&& sed -i 's/^#\?PasswordAuthentication.*/PasswordAuthentication no/' /etc/ssh/sshd_config \
|
||||
&& sed -i 's/^#\?PubkeyAuthentication.*/PubkeyAuthentication yes/' /etc/ssh/sshd_config
|
||||
|
||||
COPY keys/rsmon_test_ed25519.pub /root/.ssh/authorized_keys
|
||||
RUN chmod 600 /root/.ssh/authorized_keys
|
||||
|
||||
EXPOSE 22
|
||||
CMD ["/usr/sbin/sshd", "-D", "-e"]
|
||||
8
internal/installer/harness/testdata/keys/rsmon_test_ed25519
поставляемый
Обычный файл
8
internal/installer/harness/testdata/keys/rsmon_test_ed25519
поставляемый
Обычный файл
@@ -0,0 +1,8 @@
|
||||
-----BEGIN OPENSSH PRIVATE KEY-----
|
||||
b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAMwAAAAtzc2gtZW
|
||||
QyNTUxOQAAACCz7h3HFwJrC+DIrE3W+9yI+hRAcCVesyEQmCicmPrqAwAAAKgIO12YCDtd
|
||||
mAAAAAtzc2gtZWQyNTUxOQAAACCz7h3HFwJrC+DIrE3W+9yI+hRAcCVesyEQmCicmPrqAw
|
||||
AAAEDqbSMcuhF56miNJOZKUOuA/9I6yVga06nirb7pns41lLPuHccXAmsL4MisTdb73Ij6
|
||||
FEBwJV6zIRCYKJyY+uoDAAAAIHJzbW9uLXdvcmtlciBzb3VyY2UtaW5zdGFsbCB0ZXN0AQ
|
||||
IDBAU=
|
||||
-----END OPENSSH PRIVATE KEY-----
|
||||
1
internal/installer/harness/testdata/keys/rsmon_test_ed25519.pub
поставляемый
Обычный файл
1
internal/installer/harness/testdata/keys/rsmon_test_ed25519.pub
поставляемый
Обычный файл
@@ -0,0 +1 @@
|
||||
ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAILPuHccXAmsL4MisTdb73Ij6FEBwJV6zIRCYKJyY+uoD rsmon-worker source-install test
|
||||
@@ -478,6 +478,16 @@ func ValidateEnvironmentFile(path string) error {
|
||||
if err != nil {
|
||||
return fmt.Errorf("read worker environment: %w", err)
|
||||
}
|
||||
_, err = parseEnvironmentContent(data)
|
||||
return err
|
||||
}
|
||||
|
||||
// parseEnvironmentContent validates systemd-safe KEY=VALUE environment
|
||||
// content and returns the parsed map. Values must not use quoting,
|
||||
// interpolation, or whitespace; RSMON_URL and RSMON_TOKEN are required and
|
||||
// must not be duplicated. Operating on the already-read bytes (rather
|
||||
// than a path) keeps callers free of read-to-validate TOCTOU races.
|
||||
func parseEnvironmentContent(data []byte) (map[string]string, error) {
|
||||
values := make(map[string]string)
|
||||
seenRequired := make(map[string]bool)
|
||||
for number, line := range strings.Split(string(data), "\n") {
|
||||
@@ -486,30 +496,30 @@ func ValidateEnvironmentFile(path string) error {
|
||||
continue
|
||||
}
|
||||
if strings.ContainsRune(line, '\r') {
|
||||
return fmt.Errorf("worker environment line %d contains a carriage return", lineNumber)
|
||||
return nil, fmt.Errorf("worker environment line %d contains a carriage return", lineNumber)
|
||||
}
|
||||
key, value, ok := strings.Cut(line, "=")
|
||||
if !ok || !envKeyPattern.MatchString(key) {
|
||||
return fmt.Errorf("worker environment line %d must use KEY=VALUE syntax", lineNumber)
|
||||
return nil, fmt.Errorf("worker environment line %d must use KEY=VALUE syntax", lineNumber)
|
||||
}
|
||||
if err := validateEnvValue(key, value); err != nil {
|
||||
return fmt.Errorf("worker environment line %d: %w", lineNumber, err)
|
||||
return nil, fmt.Errorf("worker environment line %d: %w", lineNumber, err)
|
||||
}
|
||||
if key == "RSMON_URL" || key == "RSMON_TOKEN" {
|
||||
if seenRequired[key] {
|
||||
return fmt.Errorf("worker environment line %d duplicates %s", lineNumber, key)
|
||||
return nil, fmt.Errorf("worker environment line %d duplicates %s", lineNumber, key)
|
||||
}
|
||||
seenRequired[key] = true
|
||||
}
|
||||
values[key] = value
|
||||
}
|
||||
if err := ValidateURL(values["RSMON_URL"]); err != nil {
|
||||
return err
|
||||
return nil, err
|
||||
}
|
||||
if err := ValidateToken(values["RSMON_TOKEN"]); err != nil {
|
||||
return err
|
||||
return nil, err
|
||||
}
|
||||
return nil
|
||||
return values, nil
|
||||
}
|
||||
|
||||
// validateEnvValue enforces the value rules shared by the strict env
|
||||
|
||||
769
internal/installer/sourceactivate.go
Обычный файл
769
internal/installer/sourceactivate.go
Обычный файл
@@ -0,0 +1,769 @@
|
||||
package installer
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"rocketgit.ru/rsmon/worker/internal/sshinstall"
|
||||
)
|
||||
|
||||
// ActivationOptions drives work package 4 of docs/source-installation.md:
|
||||
// the atomic activation of the staged worker binary. It reuses the
|
||||
// classic installer's environment resolution (resolveInstallEnv /
|
||||
// renderEnvFile), on-disk layout (resolvePaths), and hardened systemd
|
||||
// unit renderer, then installs the binary, env file, data dir, and the
|
||||
// detected init's service definition atomically, starts/restarts the
|
||||
// worker, and verifies the process and /healthz before declaring the
|
||||
// activation a success. Any activation/start/health failure rolls back to
|
||||
// the previous working install.
|
||||
type ActivationOptions struct {
|
||||
// Activate performs the atomic install + service activation after the
|
||||
// staging build. When false SourceInstall stops at the staging build
|
||||
// (the work-package-3 boundary) and touches no service configuration.
|
||||
Activate bool
|
||||
|
||||
// Name is the instance name ("" = the primary rsmon-worker service).
|
||||
// A named instance gets rsmon-worker-<name> paths and its own unit,
|
||||
// matching the classic installer's multi-instance layout.
|
||||
Name string
|
||||
// URL is RSMON_URL (default https://rsmon.ru).
|
||||
URL string
|
||||
// Token is the worker API token (RSMON_TOKEN). Required for
|
||||
// activation; prefer supplying it through a file at the CLI layer so
|
||||
// it never appears in the process list.
|
||||
Token string
|
||||
// PublicURL is the advertised public origin (PUBLIC_URL).
|
||||
PublicURL string
|
||||
// Host is the worker webapp bind address (WORKER_HOST; default
|
||||
// 127.0.0.1 as resolved by the classic installer).
|
||||
Host string
|
||||
// Port is the worker webapp bind port (WORKER_PORT; default 27401,
|
||||
// required for named instances).
|
||||
Port string
|
||||
// Login / Password are the operator-console basic-auth credentials
|
||||
// (WORKER_LOGIN / WORKER_PASSWORD). Both must be set or both empty.
|
||||
Login string
|
||||
Password string
|
||||
// EnvFile is a local systemd-safe env file uploaded instead of the
|
||||
// individual knobs. Validated with the classic strict parser.
|
||||
EnvFile string
|
||||
// NoStart installs the binary, env, data dir, and service definition
|
||||
// without starting or restarting the worker.
|
||||
NoStart bool
|
||||
}
|
||||
|
||||
// ActivationResult records where and how the staged worker was activated.
|
||||
type ActivationResult struct {
|
||||
Binary string // installed binary path
|
||||
ConfigDir string
|
||||
EnvFile string
|
||||
DataDir string
|
||||
UnitFile string // empty when no service definition was installed
|
||||
UnitName string // systemd unit / rc-service name
|
||||
Supervisor string // "systemd", "openrc", or "supervisor" (embedded fallback)
|
||||
Started bool
|
||||
}
|
||||
|
||||
// installOptions projects the activation knobs onto the classic
|
||||
// installer's option shape so the shared env resolution is reused.
|
||||
func (a ActivationOptions) installOptions() InstallOptions {
|
||||
return InstallOptions{
|
||||
URL: a.URL,
|
||||
Token: a.Token,
|
||||
PublicURL: a.PublicURL,
|
||||
Host: a.Host,
|
||||
Port: a.Port,
|
||||
Login: a.Login,
|
||||
Password: a.Password,
|
||||
Name: strings.TrimSpace(a.Name),
|
||||
NoStart: a.NoStart,
|
||||
EnvFile: a.EnvFile,
|
||||
}
|
||||
}
|
||||
|
||||
// renderEnv resolves the full worker environment exactly as the classic
|
||||
// installer would and renders the canonical systemd-safe env file. The
|
||||
// env file (when configured) is read exactly once and validated from the
|
||||
// in-memory bytes, so a hostile local writer cannot swap the file
|
||||
// between the read used for validation and the read used for the render.
|
||||
func (a ActivationOptions) renderEnv() ([]byte, error) {
|
||||
name := strings.TrimSpace(a.Name)
|
||||
if err := validateInstanceName(name); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var fileEnv map[string]string
|
||||
if a.EnvFile != "" {
|
||||
data, err := os.ReadFile(a.EnvFile)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("--env-file: %w", err)
|
||||
}
|
||||
fe, perr := parseEnvironmentContent(data)
|
||||
if perr != nil {
|
||||
return nil, fmt.Errorf("--env-file: %w", perr)
|
||||
}
|
||||
fileEnv = fe
|
||||
}
|
||||
values, err := resolveInstallEnv(a.installOptions(), name, fileEnv)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return renderEnvFile(values), nil
|
||||
}
|
||||
|
||||
// unitContent renders the service definition for the detected init
|
||||
// system. systemd gets the classic hardened unit; OpenRC gets a native
|
||||
// init script. An unknown init produces no unit (the no-service gate:
|
||||
// the embedded supervisor still installs and manages the worker, but no
|
||||
// native service definition is written).
|
||||
func unitContent(init sshinstall.InitSystem, p paths) (content string, file string, mode string) {
|
||||
svcName := strings.TrimSuffix(p.unitName, ".service")
|
||||
switch init {
|
||||
case sshinstall.InitSystemd:
|
||||
return systemdUnitFor(p), p.unitFile, "0644"
|
||||
case sshinstall.InitOpenRC:
|
||||
return openrcInitFor(p), "/etc/init.d/" + svcName, "0755"
|
||||
default:
|
||||
return "", "", ""
|
||||
}
|
||||
}
|
||||
|
||||
// runningInitScript reports the init system that is *actually running*
|
||||
// (not merely installed): systemd, openrc, or none. The installer writes
|
||||
// the service definition for the detected init but drives start/restart
|
||||
// through whichever supervisor is usable right now; a container or chroot
|
||||
// where no init runs falls back to the embedded supervisor.
|
||||
const runningInitScript = `set -eu
|
||||
if [ -d /run/systemd/system ] && command -v systemctl >/dev/null 2>&1; then
|
||||
echo systemd
|
||||
elif [ -e /run/openrc/softlevel ] && command -v rc-service >/dev/null 2>&1; then
|
||||
echo openrc
|
||||
else
|
||||
echo none
|
||||
fi`
|
||||
|
||||
// normalizeSupervisor bounds the running-init probe output to the three
|
||||
// supervisor kinds the activation script understands.
|
||||
func normalizeSupervisor(raw string) string {
|
||||
switch strings.TrimSpace(raw) {
|
||||
case "systemd", "openrc":
|
||||
return strings.TrimSpace(raw)
|
||||
default:
|
||||
return "none"
|
||||
}
|
||||
}
|
||||
|
||||
// activateScript renders the single idempotent remote activation script.
|
||||
// It is structured as a transaction over the previous install:
|
||||
//
|
||||
// 1. acquire an activation lock so concurrent runs cannot mutate the same
|
||||
// deployment, and recover any interrupted prior activation from its
|
||||
// leftover backup marker;
|
||||
// 2. validate the staged binary (fails before any state is touched);
|
||||
// 3. snapshot the prior binary/env/unit (and the unit's enable state)
|
||||
// into a backup dir and write a recovery marker;
|
||||
// 4. arm a rollback EXIT/HUP/INT/TERM trap that restores the snapshot,
|
||||
// restores the prior enable state (or disables a freshly-installed
|
||||
// unit on a fresh failure), and restarts the prior service;
|
||||
// 5. install the new binary, env (mode 0600), data dir, and service
|
||||
// definition atomically (temp file + rename);
|
||||
// 6. start/restart through the active supervisor and verify the service
|
||||
// stays active (systemd `is-active` / rc-service `status` / the
|
||||
// embedded supervisor's zombie-aware pid check) and answers /healthz;
|
||||
// 7. on success, drop the backup and release the lock; on failure the
|
||||
// trap restores the prior install and the script exits non-zero.
|
||||
//
|
||||
// Reruns are idempotent: restart always stops the previous instance
|
||||
// first, so exactly one worker process exists, and every temp file
|
||||
// (backup dir, .new files, uploaded env/unit, lock) is removed on success
|
||||
// and failure.
|
||||
func activateScript(p activateParams) string {
|
||||
r := strings.NewReplacer(
|
||||
"@@STAGE@@", shellQuote(p.Stage),
|
||||
"@@BINARY@@", shellQuote(p.Binary),
|
||||
"@@CONFIG_DIR@@", shellQuote(p.ConfigDir),
|
||||
"@@ENV_FILE@@", shellQuote(p.EnvFile),
|
||||
"@@ENV_TMP@@", shellQuote(p.EnvTmp),
|
||||
"@@DATA_DIR@@", shellQuote(p.DataDir),
|
||||
"@@UNIT_TMP@@", shellQuote(p.UnitTmp),
|
||||
"@@UNIT_FILE@@", shellQuote(p.UnitFile),
|
||||
"@@UNIT_MODE@@", p.UnitMode,
|
||||
"@@UNIT_NAME@@", shellQuote(p.UnitName),
|
||||
"@@RC_NAME@@", shellQuote(p.RCName),
|
||||
"@@SUPERVISOR@@", p.Supervisor,
|
||||
"@@NO_START@@", p.NoStart,
|
||||
)
|
||||
return r.Replace(activateTemplate)
|
||||
}
|
||||
|
||||
// activateTemplate is the remote script body. Sentinels (@@..@@) are
|
||||
// substituted by activateScript; every operator-controlled value is
|
||||
// single-quoted and validated on the controller side first.
|
||||
const activateTemplate = `set -eu
|
||||
|
||||
stage=@@STAGE@@
|
||||
binary=@@BINARY@@
|
||||
config_dir=@@CONFIG_DIR@@
|
||||
env_file=@@ENV_FILE@@
|
||||
env_tmp=@@ENV_TMP@@
|
||||
data_dir=@@DATA_DIR@@
|
||||
unit_tmp=@@UNIT_TMP@@
|
||||
unit_file=@@UNIT_FILE@@
|
||||
unit_mode=@@UNIT_MODE@@
|
||||
unit_name=@@UNIT_NAME@@
|
||||
rc_name=@@RC_NAME@@
|
||||
supervisor=@@SUPERVISOR@@
|
||||
no_start=@@NO_START@@
|
||||
|
||||
backup_dir="$data_dir/.rsmon-backup"
|
||||
lock_dir="$data_dir/.rsmon-activate.lock"
|
||||
pid_file="$data_dir/worker.pid"
|
||||
log_file="$data_dir/worker.log"
|
||||
|
||||
run_env() {
|
||||
# Execute "$@" with a fresh environment built only from the env file
|
||||
# plus PATH, HOME, and the webapp data dir. Sourcing the env file in a
|
||||
# clean env -i shell prevents variables from a previous activation
|
||||
# (for example WORKER_CLUSTER_ENABLED) from leaking into the new
|
||||
# worker process after a rollback restores an older env file.
|
||||
env -i PATH="/usr/bin:/bin:/sbin:/usr/sbin" HOME="$data_dir" \
|
||||
RSMON_WEBAPP_DATA_DIR="$data_dir/webapp" \
|
||||
RSMON_WORKER_ENV_FILE="$env_file" sh -c '
|
||||
set -a
|
||||
. "$RSMON_WORKER_ENV_FILE"
|
||||
set +a
|
||||
unset RSMON_WORKER_ENV_FILE
|
||||
exec "$@"
|
||||
' sh "$@"
|
||||
}
|
||||
|
||||
process_up() {
|
||||
# True when the given pid is a live, non-zombie process. A zombie
|
||||
# still answers kill -0 (its task struct exists until reaped), and in
|
||||
# a container where PID 1 does not reap children a killed worker can
|
||||
# linger as a zombie for a long time, so the /proc state must be
|
||||
# checked explicitly.
|
||||
[ -n "$1" ] || return 1
|
||||
kill -0 "$1" 2>/dev/null || return 1
|
||||
state="$(awk '{print $3}' "/proc/$1/stat" 2>/dev/null || true)"
|
||||
[ "$state" != "Z" ] || return 1
|
||||
}
|
||||
|
||||
process_is_worker() {
|
||||
# True when the given pid's executable is (or was, before an atomic
|
||||
# binary swap) the configured worker binary, so a stale or recycled
|
||||
# pid file can never make the supervisor kill an unrelated process.
|
||||
[ -n "$1" ] || return 1
|
||||
exe="$(readlink "/proc/$1/exe" 2>/dev/null || true)"
|
||||
case "$exe" in
|
||||
"$binary"|"$binary (deleted)") return 0 ;;
|
||||
esac
|
||||
return 1
|
||||
}
|
||||
|
||||
stop_process() {
|
||||
if [ -f "$pid_file" ]; then
|
||||
pid="$(cat "$pid_file" 2>/dev/null || true)"
|
||||
if [ -n "$pid" ] && process_is_worker "$pid" && process_up "$pid"; then
|
||||
kill "$pid" 2>/dev/null || true
|
||||
i=0
|
||||
while [ "$i" -lt 10 ]; do
|
||||
if ! process_up "$pid"; then
|
||||
break
|
||||
fi
|
||||
sleep 1
|
||||
i=$((i + 1))
|
||||
done
|
||||
kill -9 "$pid" 2>/dev/null || true
|
||||
fi
|
||||
fi
|
||||
rm -f "$pid_file"
|
||||
}
|
||||
|
||||
start_process() {
|
||||
stop_process
|
||||
mkdir -p "$data_dir"
|
||||
run_env sh -c 'nohup "$1" >>"$2" 2>&1 & echo $! > "$3"' sh "$binary" "$log_file" "$pid_file"
|
||||
}
|
||||
|
||||
process_alive() {
|
||||
[ -f "$pid_file" ] || return 1
|
||||
pid="$(cat "$pid_file" 2>/dev/null || true)"
|
||||
process_is_worker "$pid" || return 1
|
||||
process_up "$pid"
|
||||
}
|
||||
|
||||
worker_come_up() {
|
||||
# Wait up to 10s for the freshly-started worker to become a live,
|
||||
# non-zombie process running the configured binary. The recorded pid
|
||||
# starts as the nohup/sh child before it execs the worker, so early
|
||||
# readlink checks can transiently see the interpreter; this retry both
|
||||
# tolerates that exec window and catches genuine immediate deaths.
|
||||
i=0
|
||||
while [ "$i" -lt 10 ]; do
|
||||
if process_alive; then
|
||||
return 0
|
||||
fi
|
||||
sleep 1
|
||||
i=$((i + 1))
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
health_ok() {
|
||||
run_env "$binary" liveness >/dev/null 2>&1
|
||||
}
|
||||
|
||||
svc_active() {
|
||||
# The health loop verifies the service is active through the
|
||||
# supervisor (never through a pid file for the init-managed paths).
|
||||
case "$supervisor" in
|
||||
systemd) systemctl is-active --quiet "$unit_name" ;;
|
||||
openrc) rc-service "$rc_name" status >/dev/null 2>&1 ;;
|
||||
none) process_alive ;;
|
||||
esac
|
||||
}
|
||||
|
||||
stop_svc() {
|
||||
case "$supervisor" in
|
||||
systemd) systemctl stop "$unit_name" >/dev/null 2>&1 || true ;;
|
||||
openrc) rc-service "$rc_name" stop >/dev/null 2>&1 || true ;;
|
||||
none) stop_process ;;
|
||||
esac
|
||||
}
|
||||
|
||||
restart_svc() {
|
||||
case "$supervisor" in
|
||||
systemd)
|
||||
systemctl daemon-reload >/dev/null 2>&1
|
||||
if ! systemctl restart "$unit_name" >/dev/null 2>&1; then
|
||||
echo "start failure: systemctl restart $unit_name failed" >&2
|
||||
return 1
|
||||
fi
|
||||
if ! systemctl is-active --quiet "$unit_name"; then
|
||||
echo "start failure: unit $unit_name is not active" >&2
|
||||
return 1
|
||||
fi
|
||||
;;
|
||||
openrc)
|
||||
if ! rc-service "$rc_name" restart >/dev/null 2>&1; then
|
||||
echo "start failure: rc-service restart $rc_name failed" >&2
|
||||
return 1
|
||||
fi
|
||||
if ! rc-service "$rc_name" status >/dev/null 2>&1; then
|
||||
echo "start failure: service $rc_name is not running" >&2
|
||||
return 1
|
||||
fi
|
||||
;;
|
||||
none)
|
||||
start_process
|
||||
if ! worker_come_up; then
|
||||
echo "start failure: worker process did not stay alive" >&2
|
||||
return 1
|
||||
fi
|
||||
;;
|
||||
esac
|
||||
i=0
|
||||
while [ "$i" -lt 30 ]; do
|
||||
if health_ok; then
|
||||
return 0
|
||||
fi
|
||||
# A worker that stops while the health gate is still probing is a
|
||||
# start failure (it never became a stable service), not merely an
|
||||
# unhealthy-but-running one.
|
||||
if ! svc_active; then
|
||||
echo "start failure: worker service is not active" >&2
|
||||
return 1
|
||||
fi
|
||||
sleep 1
|
||||
i=$((i + 1))
|
||||
done
|
||||
echo "health failure: worker did not answer /healthz" >&2
|
||||
return 1
|
||||
}
|
||||
|
||||
# --- unit enable helpers -------------------------------------------------
|
||||
# unit_was_enabled records whether the unit referenced by the current
|
||||
# params is enabled (best-effort; a missing init leaves it disabled).
|
||||
unit_was_enabled() {
|
||||
case "$supervisor" in
|
||||
systemd) systemctl is-enabled "$unit_name" >/dev/null 2>&1 && unit_enabled=1 ;;
|
||||
openrc) [ -e "/etc/runlevels/default/$rc_name" ] && unit_enabled=1 ;;
|
||||
esac
|
||||
return 0
|
||||
}
|
||||
# set_unit_enabled "$1" enables (1) or disables (0) the unit.
|
||||
set_unit_enabled() {
|
||||
case "$supervisor" in
|
||||
systemd)
|
||||
systemctl daemon-reload >/dev/null 2>&1 || true
|
||||
if [ "$1" -eq 1 ]; then
|
||||
systemctl enable "$unit_name" >/dev/null 2>&1 || true
|
||||
else
|
||||
systemctl disable "$unit_name" >/dev/null 2>&1 || true
|
||||
fi
|
||||
;;
|
||||
openrc)
|
||||
if [ "$1" -eq 1 ]; then
|
||||
rc-update add "$rc_name" default >/dev/null 2>&1 || true
|
||||
else
|
||||
rc-update del "$rc_name" default >/dev/null 2>&1 || true
|
||||
fi
|
||||
;;
|
||||
esac
|
||||
return 0
|
||||
}
|
||||
|
||||
# --- activation lock ------------------------------------------------------
|
||||
# Prevent concurrent activations on the same host from racing the
|
||||
# deployed-state mutation. A stale lock (left over from a SIGKILL'd run
|
||||
# whose pid is no longer alive) is broken automatically.
|
||||
acquire_lock() {
|
||||
mkdir -p "$data_dir"
|
||||
if [ -d "$lock_dir" ]; then
|
||||
lockpid="$(cat "$lock_dir/pid" 2>/dev/null || true)"
|
||||
if [ -n "$lockpid" ] && ! kill -0 "$lockpid" 2>/dev/null; then
|
||||
rm -rf "$lock_dir" 2>/dev/null || true
|
||||
fi
|
||||
fi
|
||||
i=0
|
||||
while [ "$i" -lt 60 ]; do
|
||||
if mkdir "$lock_dir" 2>/dev/null; then
|
||||
chmod 0700 "$lock_dir" 2>/dev/null || true
|
||||
echo $$ > "$lock_dir/pid" 2>/dev/null || true
|
||||
return 0
|
||||
fi
|
||||
sleep 1
|
||||
i=$((i + 1))
|
||||
done
|
||||
echo "another rsmon-worker activation is in progress ($lock_dir)" >&2
|
||||
return 1
|
||||
}
|
||||
release_lock() {
|
||||
rm -rf "$lock_dir" 2>/dev/null || true
|
||||
}
|
||||
|
||||
if ! acquire_lock; then
|
||||
exit 1
|
||||
fi
|
||||
# Any early failure (before the rollback trap is armed) still releases the
|
||||
# lock.
|
||||
trap 'release_lock' EXIT
|
||||
|
||||
# --- interrupted-run recovery --------------------------------------------
|
||||
# A previous activation that was killed mid-flight (SSH drop, SIGKILL)
|
||||
# leaves its backup marker behind. Restore that snapshot before the fresh
|
||||
# activation runs, so the host never stays half-activated and the next run
|
||||
# starts from a consistent prior state.
|
||||
unit_enabled=0
|
||||
recover_backup() {
|
||||
if [ ! -f "$backup_dir/.marker" ]; then
|
||||
return 0
|
||||
fi
|
||||
echo "recovering interrupted activation from $backup_dir" >&2
|
||||
if [ -f "$backup_dir/binary" ]; then
|
||||
mkdir -p "$(dirname "$binary")" || true
|
||||
cp -p "$backup_dir/binary" "$binary" || true
|
||||
fi
|
||||
if [ -f "$backup_dir/worker.env" ]; then
|
||||
mkdir -p "$config_dir" || true
|
||||
cp -p "$backup_dir/worker.env" "$env_file" || true
|
||||
fi
|
||||
if [ -n "$unit_file" ] && [ -f "$backup_dir/unit" ]; then
|
||||
mkdir -p "$(dirname "$unit_file")" || true
|
||||
cp -p "$backup_dir/unit" "$unit_file" || true
|
||||
chmod "$unit_mode" "$unit_file" || true
|
||||
fi
|
||||
if [ "$(cat "$backup_dir/.marker" 2>/dev/null || true)" = "1" ]; then
|
||||
unit_enabled=1
|
||||
fi
|
||||
set_unit_enabled "$unit_enabled" || true
|
||||
if [ -f "$backup_dir/binary" ] && [ -f "$backup_dir/worker.env" ] && [ "$no_start" -ne 1 ]; then
|
||||
restart_svc || true
|
||||
fi
|
||||
rm -rf "$backup_dir" || true
|
||||
}
|
||||
recover_backup
|
||||
|
||||
# --- staged binary validation (no state touched) -------------------------
|
||||
# A corrupt or missing staging build aborts here and leaves the prior
|
||||
# install completely untouched (the rollback trap is not armed yet).
|
||||
"$stage" --version >/dev/null
|
||||
|
||||
# --- snapshot the prior install ------------------------------------------
|
||||
rm -rf "$backup_dir"
|
||||
mkdir -p "$backup_dir"
|
||||
had_binary=0
|
||||
had_env=0
|
||||
had_unit=0
|
||||
unit_enabled=0
|
||||
if [ -e "$binary" ]; then
|
||||
cp -p "$binary" "$backup_dir/binary"
|
||||
had_binary=1
|
||||
fi
|
||||
if [ -e "$env_file" ]; then
|
||||
cp -p "$env_file" "$backup_dir/worker.env"
|
||||
had_env=1
|
||||
fi
|
||||
if [ -n "$unit_file" ] && [ -e "$unit_file" ]; then
|
||||
had_unit=1
|
||||
cp -p "$unit_file" "$backup_dir/unit"
|
||||
unit_was_enabled
|
||||
fi
|
||||
# The marker is written only after the snapshot copies so recovery never
|
||||
# sees a partial snapshot; it records whether the prior unit was enabled.
|
||||
printf '%s\n' "$unit_enabled" > "$backup_dir/.marker"
|
||||
|
||||
# --- rollback trap --------------------------------------------------------
|
||||
# Any failure from here on restores the snapshot (preserving binary/env/unit
|
||||
# metadata), restores the prior enable state (or disables a freshly-installed
|
||||
# unit on a fresh install), and brings the prior service back up. The trap
|
||||
# also fires on HUP/INT/TERM so an interrupted run rolls back instead of
|
||||
# leaving a half-activated state.
|
||||
rolled_back=0
|
||||
rollback() {
|
||||
[ "$rolled_back" -eq 1 ] && return 0
|
||||
rolled_back=1
|
||||
echo "rsmon-worker activation failed; restoring the prior install" >&2
|
||||
stop_svc || true
|
||||
rm -f "$binary.new" "$env_file.new" "$unit_file.new" "$pid_file" || true
|
||||
if [ "$had_binary" -eq 1 ]; then
|
||||
cp -p "$backup_dir/binary" "$binary" || true
|
||||
elif [ -e "$binary" ]; then
|
||||
rm -f "$binary" || true
|
||||
fi
|
||||
if [ "$had_env" -eq 1 ]; then
|
||||
cp -p "$backup_dir/worker.env" "$env_file" || true
|
||||
chmod 0600 "$env_file" || true
|
||||
elif [ -e "$env_file" ]; then
|
||||
rm -f "$env_file" || true
|
||||
fi
|
||||
if [ -n "$unit_file" ]; then
|
||||
if [ "$had_unit" -eq 1 ]; then
|
||||
cp -p "$backup_dir/unit" "$unit_file" || true
|
||||
chmod "$unit_mode" "$unit_file" || true
|
||||
set_unit_enabled "$unit_enabled" || true
|
||||
else
|
||||
set_unit_enabled 0 || true
|
||||
rm -f "$unit_file" || true
|
||||
fi
|
||||
fi
|
||||
rm -f "$env_tmp" "$unit_tmp" || true
|
||||
rm -rf "$backup_dir" || true
|
||||
if [ "$had_binary" -eq 1 ] && [ "$had_env" -eq 1 ] && [ "$no_start" -ne 1 ]; then
|
||||
restart_svc || true
|
||||
fi
|
||||
release_lock
|
||||
exit 1
|
||||
}
|
||||
trap rollback EXIT HUP INT TERM
|
||||
|
||||
# --- atomic binary install ------------------------------------------------
|
||||
mkdir -p "$(dirname "$binary")"
|
||||
install -m 0755 "$stage" "$binary.new"
|
||||
mv -f "$binary.new" "$binary"
|
||||
|
||||
# --- atomic env install (secrets, mode 0600) ------------------------------
|
||||
mkdir -p "$config_dir"
|
||||
chmod 0750 "$config_dir"
|
||||
install -m 0600 "$env_tmp" "$env_file.new"
|
||||
mv -f "$env_file.new" "$env_file"
|
||||
rm -f "$env_tmp"
|
||||
|
||||
# --- data directory -------------------------------------------------------
|
||||
mkdir -p "$data_dir" "$data_dir/webapp"
|
||||
chmod 0755 "$data_dir" "$data_dir/webapp"
|
||||
|
||||
# --- service definition ---------------------------------------------------
|
||||
if [ -n "$unit_file" ] && [ -f "$unit_tmp" ]; then
|
||||
mkdir -p "$(dirname "$unit_file")"
|
||||
install -m "$unit_mode" "$unit_tmp" "$unit_file.new"
|
||||
mv -f "$unit_file.new" "$unit_file"
|
||||
rm -f "$unit_tmp"
|
||||
set_unit_enabled 1
|
||||
fi
|
||||
|
||||
# --- start and verify -----------------------------------------------------
|
||||
# A failed start or a worker that does not answer /healthz triggers the
|
||||
# rollback trap.
|
||||
if [ "$no_start" -ne 1 ]; then
|
||||
if ! restart_svc; then
|
||||
echo "rsmon-worker activation failed: process or /healthz verification failed" >&2
|
||||
exit 1
|
||||
fi
|
||||
else
|
||||
stop_svc || true
|
||||
fi
|
||||
|
||||
# --- success --------------------------------------------------------------
|
||||
rm -rf "$backup_dir"
|
||||
rm -f "$env_tmp" "$unit_tmp"
|
||||
release_lock
|
||||
trap - EXIT HUP INT TERM
|
||||
if [ "$no_start" -eq 1 ]; then
|
||||
echo "rsmon-worker activated: binary=$binary supervisor=$supervisor unit=${unit_file:-none} started=no"
|
||||
else
|
||||
echo "rsmon-worker activated: binary=$binary supervisor=$supervisor unit=${unit_file:-none} started=yes"
|
||||
fi
|
||||
exit 0
|
||||
`
|
||||
|
||||
// activateParams are the resolved, controller-validated values fed into
|
||||
// the remote activation script.
|
||||
type activateParams struct {
|
||||
Stage string
|
||||
Binary string
|
||||
ConfigDir string
|
||||
EnvFile string
|
||||
EnvTmp string
|
||||
DataDir string
|
||||
UnitTmp string
|
||||
UnitFile string
|
||||
UnitMode string
|
||||
UnitName string // systemd unit name (rsmon-worker.service)
|
||||
RCName string // rc-service name (rsmon-worker)
|
||||
Supervisor string // systemd | openrc | none
|
||||
NoStart string // 0 or 1
|
||||
}
|
||||
|
||||
// activateWorker performs the work-package-4 activation for an already
|
||||
// staged source build over the live SSH executor. It uses the env bytes
|
||||
// rendered once during option normalization, uploads the env and service
|
||||
// definition into a server-created 0700 temp dir (no /tmp symlink/TOCTOU
|
||||
// attack surface), runs the atomic activation script, and records where
|
||||
// and how the worker was activated. Every remote temp file it creates is
|
||||
// removed on success and failure.
|
||||
func (e *sourceExecutor) activateWorker(opts SourceInstallOptions, res *SourceInstallResult) error {
|
||||
a := opts.Activation
|
||||
name := strings.TrimSpace(a.Name)
|
||||
p := resolvePaths(name)
|
||||
|
||||
envData := opts.activationEnv
|
||||
if len(envData) == 0 {
|
||||
return errors.New("activation environment was not rendered (internal error)")
|
||||
}
|
||||
|
||||
unitContent, unitFile, unitMode := unitContent(res.Detection.InitSystem, p)
|
||||
svcName := strings.TrimSuffix(p.unitName, ".service")
|
||||
|
||||
supOut, err := e.runPlain("sh -c " + shellQuote(runningInitScript))
|
||||
if err != nil {
|
||||
return fmt.Errorf("detect running init system: %w", err)
|
||||
}
|
||||
supervisor := normalizeSupervisor(string(supOut))
|
||||
|
||||
// Create a server-side 0700 temp dir owned by the SSH user, then
|
||||
// upload the env/unit into it. mktemp -d produces an unpredictable,
|
||||
// private path, so a hostile local user cannot pre-create a symlink at
|
||||
// a predictable /tmp name (the classic upload TOCTOU). The token
|
||||
// never appears in argv or logs: it travels only as base64 over the
|
||||
// session stdin and later only inside the mode-0600 env file.
|
||||
out, err := e.runPlain("d=$(mktemp -d /tmp/rsmon-worker-act.XXXXXX) && chmod 0700 \"$d\" && echo \"$d\"")
|
||||
if err != nil {
|
||||
return fmt.Errorf("create secure upload directory: %w", err)
|
||||
}
|
||||
secureDir := strings.TrimSpace(string(out))
|
||||
if secureDir == "" || !strings.HasPrefix(secureDir, "/tmp/") {
|
||||
return errors.New("remote returned an invalid secure upload directory")
|
||||
}
|
||||
envTmp := secureDir + "/worker.env"
|
||||
unitTmp := secureDir + "/unit"
|
||||
if err := uploadBytes(e.client, envData, envTmp, 0o600); err != nil {
|
||||
return fmt.Errorf("upload worker environment: %w", err)
|
||||
}
|
||||
if unitContent != "" {
|
||||
if err := uploadBytes(e.client, []byte(unitContent), unitTmp, 0o644); err != nil {
|
||||
return fmt.Errorf("upload service definition: %w", err)
|
||||
}
|
||||
}
|
||||
defer func() {
|
||||
_ = runRemote(e.client, "rm -rf -- "+shellQuote(secureDir), nil)
|
||||
}()
|
||||
|
||||
noStart := "0"
|
||||
if a.NoStart {
|
||||
noStart = "1"
|
||||
}
|
||||
script := activateScript(activateParams{
|
||||
Stage: res.StageBinary,
|
||||
Binary: p.binary,
|
||||
ConfigDir: p.configDir,
|
||||
EnvFile: p.envFile,
|
||||
EnvTmp: envTmp,
|
||||
DataDir: p.dataDir,
|
||||
UnitTmp: unitTmp,
|
||||
UnitFile: unitFile,
|
||||
UnitMode: unitMode,
|
||||
UnitName: p.unitName,
|
||||
RCName: svcName,
|
||||
Supervisor: supervisor,
|
||||
NoStart: noStart,
|
||||
})
|
||||
if _, err := e.runPrivileged("sh -c " + shellQuote(script)); err != nil {
|
||||
return fmt.Errorf("activate worker service: %w", err)
|
||||
}
|
||||
|
||||
res.Activation = &ActivationResult{
|
||||
Binary: p.binary,
|
||||
ConfigDir: p.configDir,
|
||||
EnvFile: p.envFile,
|
||||
DataDir: p.dataDir,
|
||||
UnitFile: unitFile,
|
||||
UnitName: p.unitName,
|
||||
Supervisor: supervisor,
|
||||
Started: !a.NoStart,
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// validateRenderedEnv runs the strict environment-file parser over the
|
||||
// rendered env bytes in memory so the exact bytes written remotely pass
|
||||
// the same gate as a user-supplied file, without re-reading a file.
|
||||
func validateRenderedEnv(data []byte) error {
|
||||
_, err := parseEnvironmentContent(data)
|
||||
return err
|
||||
}
|
||||
|
||||
// openrcInitFor renders the native OpenRC init script for an instance. It
|
||||
// mirrors the hardening of the systemd unit (data dir, env file, webapp
|
||||
// data dir) using OpenRC conventions; it is written for real hosts where
|
||||
// openrc is the running init, while containers that cannot run openrc
|
||||
// fall back to the embedded supervisor.
|
||||
func openrcInitFor(p paths) string {
|
||||
description := "RSMon distributed monitoring worker"
|
||||
svcName := strings.TrimSuffix(p.unitName, ".service")
|
||||
if p.name != "" {
|
||||
description += " (" + p.name + ")"
|
||||
}
|
||||
return fmt.Sprintf(`#!/sbin/openrc-run
|
||||
# Managed by the rsmon-worker source installer; do not edit by hand.
|
||||
|
||||
name=%s
|
||||
description=%s
|
||||
command=%s
|
||||
command_background=true
|
||||
pidfile=%s/worker.pid
|
||||
output_log=%s/worker.log
|
||||
error_log=%s/worker.log
|
||||
|
||||
depend() {
|
||||
need net
|
||||
}
|
||||
|
||||
start_pre() {
|
||||
checkpath --directory --mode 0755 --owner root:root %s %s/webapp
|
||||
if [ -f %s ]; then
|
||||
# OpenRC runs the command with the init script's environment, so
|
||||
# the env-file variables must be exported (set -a) or the worker
|
||||
# would start without them.
|
||||
set -a
|
||||
. %s
|
||||
set +a
|
||||
fi
|
||||
export HOME=%s
|
||||
export RSMON_WEBAPP_DATA_DIR=%s/webapp
|
||||
}
|
||||
`, svcName, description, p.binary, p.dataDir, p.dataDir, p.dataDir,
|
||||
p.dataDir, p.dataDir, p.envFile, p.envFile, p.dataDir, p.dataDir)
|
||||
}
|
||||
438
internal/installer/sourceactivate_shell_test.go
Обычный файл
438
internal/installer/sourceactivate_shell_test.go
Обычный файл
@@ -0,0 +1,438 @@
|
||||
package installer
|
||||
|
||||
import (
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// This file exercises the rendered activation script against stub
|
||||
// systemd/OpenRC implementations ("targeted shell fixture tests"). Real
|
||||
// systemd and OpenRC cannot run inside the Docker/OpenSSH harness (the
|
||||
// container PID 1 is sshd), so these tests execute the actual remote
|
||||
// script on the host with fake `systemctl`, `rc-service`, and `rc-update`
|
||||
// binaries that simulate an init-managed host. This covers the unit
|
||||
// install/enable, the supervisor-driven restart, the `is-active`-based
|
||||
// health loop, the enable-state rollback, the activation lock, and the
|
||||
// interrupted-run recovery paths that the E2E fixtures cannot reach.
|
||||
//
|
||||
// Residual limitation: the stub init tools verify command flow and state
|
||||
// transitions, not the real systemd/OpenRC unit semantics; a real
|
||||
// init-system smoke test remains out of scope for the container harness.
|
||||
|
||||
// systemctlStub records every invocation and simulates unit state in a
|
||||
// fake state directory: `is-active`/`is-enabled` consult marker files,
|
||||
// `restart` marks the unit active (or fails when the fail-restart marker
|
||||
// is present), and `enable`/`disable` toggle the enabled marker.
|
||||
const systemctlStub = `#!/bin/sh
|
||||
echo "$*" >> "$SYSCTL_LOG"
|
||||
action="$1"
|
||||
shift
|
||||
unit=""
|
||||
for a in "$@"; do
|
||||
case "$a" in --*) continue ;; esac
|
||||
unit="$a"
|
||||
break
|
||||
done
|
||||
case "$action" in
|
||||
is-active) [ -f "$FAKE_SYSTEMD_DIR/$unit.active" ] || exit 3 ;;
|
||||
is-enabled) [ -f "$FAKE_SYSTEMD_DIR/$unit.enabled" ] || exit 1 ;;
|
||||
enable) touch "$FAKE_SYSTEMD_DIR/$unit.enabled"; exit 0 ;;
|
||||
disable) rm -f "$FAKE_SYSTEMD_DIR/$unit.enabled"; exit 0 ;;
|
||||
daemon-reload) exit 0 ;;
|
||||
restart) [ -f "$FAKE_SYSTEMD_DIR/fail-restart" ] && exit 1
|
||||
touch "$FAKE_SYSTEMD_DIR/$unit.active"; exit 0 ;;
|
||||
stop) rm -f "$FAKE_SYSTEMD_DIR/$unit.active"; exit 0 ;;
|
||||
*) exit 0 ;;
|
||||
esac
|
||||
`
|
||||
|
||||
// rcServiceStub simulates OpenRC service state; rcUpdateStub simulates
|
||||
// runlevel enablement.
|
||||
const rcServiceStub = `#!/bin/sh
|
||||
echo "$*" >> "$RC_LOG"
|
||||
svc="$1"
|
||||
action="$2"
|
||||
case "$action" in
|
||||
restart) [ -f "$FAKE_RC_DIR/fail-restart" ] && exit 1
|
||||
touch "$FAKE_RC_DIR/$svc.active"; exit 0 ;;
|
||||
status) [ -f "$FAKE_RC_DIR/$svc.active" ] || exit 1 ;;
|
||||
stop) rm -f "$FAKE_RC_DIR/$svc.active"; exit 0 ;;
|
||||
*) exit 0 ;;
|
||||
esac
|
||||
`
|
||||
|
||||
const rcUpdateStub = `#!/bin/sh
|
||||
echo "$*" >> "$RCU_LOG"
|
||||
case "$1" in
|
||||
add) touch "$FAKE_RC_DIR/$2.enabled"; exit 0 ;;
|
||||
del) rm -f "$FAKE_RC_DIR/$2.enabled"; exit 0 ;;
|
||||
*) exit 0 ;;
|
||||
esac
|
||||
`
|
||||
|
||||
// workerStub stands in for the built worker binary: it satisfies the
|
||||
// `--version` validation and the `/healthz` liveness probe.
|
||||
const workerStub = `#!/bin/sh
|
||||
if [ "$1" = "--version" ]; then
|
||||
echo "rsmon-worker version=dev commit=deadbeefdead buildDate=2026-01-01T00:00:00Z"
|
||||
exit 0
|
||||
fi
|
||||
if [ "$1" = "liveness" ]; then
|
||||
exit 0
|
||||
fi
|
||||
exit 1
|
||||
`
|
||||
|
||||
// initShellFixture builds a temp "host" with stub init tools and a stub
|
||||
// worker, plus the uploaded env/unit the activation script consumes.
|
||||
type initShellFixture struct {
|
||||
root string
|
||||
binDir string
|
||||
sysdDir string
|
||||
rcDir string
|
||||
|
||||
sysctlLog string
|
||||
rcLog string
|
||||
rcuLog string
|
||||
|
||||
params activateParams
|
||||
}
|
||||
|
||||
func writeExec(t *testing.T, path, content string) {
|
||||
t.Helper()
|
||||
if err := os.WriteFile(path, []byte(content), 0o700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func newInitShellFixture(t *testing.T, supervisor string) *initShellFixture {
|
||||
t.Helper()
|
||||
root := t.TempDir()
|
||||
fx := &initShellFixture{
|
||||
root: root,
|
||||
binDir: filepath.Join(root, "bin"),
|
||||
sysdDir: filepath.Join(root, "sysd"),
|
||||
rcDir: filepath.Join(root, "rc"),
|
||||
sysctlLog: filepath.Join(root, "sysctl.log"),
|
||||
rcLog: filepath.Join(root, "rc.log"),
|
||||
rcuLog: filepath.Join(root, "rcu.log"),
|
||||
}
|
||||
for _, d := range []string{
|
||||
fx.binDir, fx.sysdDir, fx.rcDir, filepath.Join(root, "upload"),
|
||||
filepath.Join(root, "usr/local/bin"), filepath.Join(root, "etc/rsmon-worker"),
|
||||
} {
|
||||
if err := os.MkdirAll(d, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
writeExec(t, filepath.Join(fx.binDir, "systemctl"), systemctlStub)
|
||||
writeExec(t, filepath.Join(fx.binDir, "rc-service"), rcServiceStub)
|
||||
writeExec(t, filepath.Join(fx.binDir, "rc-update"), rcUpdateStub)
|
||||
|
||||
stage := filepath.Join(root, "usr/local/bin/rsmon-worker.stage")
|
||||
writeExec(t, stage, workerStub)
|
||||
|
||||
envTmp := filepath.Join(root, "upload", "worker.env")
|
||||
unitTmp := filepath.Join(root, "upload", "unit")
|
||||
if err := os.WriteFile(envTmp, []byte(
|
||||
"RSMON_URL=https://rsmon.ru\nRSMON_TOKEN=shell-test-token\nWORKER_HOST=127.0.0.1\nWORKER_PORT=27401\n",
|
||||
), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
unitFile := filepath.Join(root, "etc/systemd/system/rsmon-worker.service")
|
||||
unitMode := "0644"
|
||||
if supervisor == "openrc" {
|
||||
unitFile = filepath.Join(root, "etc/init.d/rsmon-worker")
|
||||
unitMode = "0755"
|
||||
if err := os.MkdirAll(filepath.Dir(unitFile), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(unitTmp, []byte("# fake openrc unit\n"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
} else {
|
||||
if err := os.MkdirAll(filepath.Dir(unitFile), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(unitTmp, []byte("[Unit]\n# fake systemd unit\n"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
dataDir := filepath.Join(root, "var/lib/rsmon-worker")
|
||||
fx.params = activateParams{
|
||||
Stage: stage,
|
||||
Binary: filepath.Join(root, "usr/local/bin/rsmon-worker"),
|
||||
ConfigDir: filepath.Join(root, "etc/rsmon-worker"),
|
||||
EnvFile: filepath.Join(root, "etc/rsmon-worker/worker.env"),
|
||||
EnvTmp: envTmp,
|
||||
DataDir: dataDir,
|
||||
UnitTmp: unitTmp,
|
||||
UnitFile: unitFile,
|
||||
UnitMode: unitMode,
|
||||
UnitName: "rsmon-worker.service",
|
||||
RCName: "rsmon-worker",
|
||||
Supervisor: supervisor,
|
||||
NoStart: "0",
|
||||
}
|
||||
return fx
|
||||
}
|
||||
|
||||
// run executes the rendered activation script with the stub init tools
|
||||
// first in PATH.
|
||||
func (fx *initShellFixture) run(t *testing.T) (string, error) {
|
||||
t.Helper()
|
||||
script := activateScript(fx.params)
|
||||
cmd := exec.Command("sh", "-c", script)
|
||||
cmd.Env = append(
|
||||
os.Environ(),
|
||||
"PATH="+fx.binDir+":"+os.Getenv("PATH"),
|
||||
"FAKE_SYSTEMD_DIR="+fx.sysdDir,
|
||||
"FAKE_RC_DIR="+fx.rcDir,
|
||||
"SYSCTL_LOG="+fx.sysctlLog,
|
||||
"RC_LOG="+fx.rcLog,
|
||||
"RCU_LOG="+fx.rcuLog,
|
||||
)
|
||||
out, err := cmd.CombinedOutput()
|
||||
return string(out), err
|
||||
}
|
||||
|
||||
func (fx *initShellFixture) log(t *testing.T, path string) string {
|
||||
t.Helper()
|
||||
b, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
return string(b)
|
||||
}
|
||||
|
||||
func (fx *initShellFixture) assertNoLeftovers(t *testing.T) {
|
||||
t.Helper()
|
||||
for _, name := range []string{".rsmon-backup", ".rsmon-activate.lock"} {
|
||||
if _, err := os.Stat(filepath.Join(fx.params.DataDir, name)); !os.IsNotExist(err) {
|
||||
t.Fatalf("%s left behind after activation", name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestActivateShellSystemd runs the real activation script against a stub
|
||||
// systemd host and verifies the init-managed flow: the unit is installed
|
||||
// and enabled, restart goes through systemctl, and the health loop uses
|
||||
// `systemctl is-active` (no pid file is ever created).
|
||||
func TestActivateShellSystemd(t *testing.T) {
|
||||
fx := newInitShellFixture(t, "systemd")
|
||||
out, err := fx.run(t)
|
||||
if err != nil {
|
||||
t.Fatalf("systemd activation failed: %v\n%s", err, out)
|
||||
}
|
||||
if !strings.Contains(out, "rsmon-worker activated") {
|
||||
t.Fatalf("no activation success line:\n%s", out)
|
||||
}
|
||||
unit, err := os.ReadFile(fx.params.UnitFile)
|
||||
if err != nil {
|
||||
t.Fatalf("unit not installed: %v", err)
|
||||
}
|
||||
if !strings.Contains(string(unit), "[Unit]") {
|
||||
t.Fatalf("installed unit has wrong content: %q", unit)
|
||||
}
|
||||
st, err := os.Stat(fx.params.EnvFile)
|
||||
if err != nil {
|
||||
t.Fatalf("env not installed: %v", err)
|
||||
}
|
||||
if st.Mode().Perm() != 0o600 {
|
||||
t.Fatalf("env mode = %v, want 0600", st.Mode().Perm())
|
||||
}
|
||||
log := fx.log(t, fx.sysctlLog)
|
||||
for _, want := range []string{
|
||||
"enable rsmon-worker.service",
|
||||
"restart rsmon-worker.service",
|
||||
"is-active --quiet rsmon-worker.service",
|
||||
} {
|
||||
if !strings.Contains(log, want) {
|
||||
t.Fatalf("systemctl log missing %q:\n%s", want, log)
|
||||
}
|
||||
}
|
||||
// The health loop used systemctl is-active, never the embedded
|
||||
// supervisor's pid file.
|
||||
if _, err := os.Stat(filepath.Join(fx.params.DataDir, "worker.pid")); !os.IsNotExist(err) {
|
||||
t.Fatalf("systemd activation created a pid file; the health loop must use systemctl is-active")
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(fx.sysdDir, "rsmon-worker.service.enabled")); err != nil {
|
||||
t.Fatalf("unit not enabled: %v", err)
|
||||
}
|
||||
fx.assertNoLeftovers(t)
|
||||
}
|
||||
|
||||
// TestActivateShellOpenRC runs the real activation script against a stub
|
||||
// OpenRC host: the init script is installed executable, the runlevel
|
||||
// enablement and rc-service restart/status are used, and the health loop
|
||||
// goes through rc-service (no pid file).
|
||||
func TestActivateShellOpenRC(t *testing.T) {
|
||||
fx := newInitShellFixture(t, "openrc")
|
||||
out, err := fx.run(t)
|
||||
if err != nil {
|
||||
t.Fatalf("openrc activation failed: %v\n%s", err, out)
|
||||
}
|
||||
if !strings.Contains(out, "rsmon-worker activated") {
|
||||
t.Fatalf("no activation success line:\n%s", out)
|
||||
}
|
||||
st, err := os.Stat(fx.params.UnitFile)
|
||||
if err != nil {
|
||||
t.Fatalf("openrc init not installed: %v", err)
|
||||
}
|
||||
if st.Mode().Perm() != 0o755 {
|
||||
t.Fatalf("openrc init mode = %v, want 0755", st.Mode().Perm())
|
||||
}
|
||||
rcLog := fx.log(t, fx.rcLog)
|
||||
for _, want := range []string{"rsmon-worker restart", "rsmon-worker status"} {
|
||||
if !strings.Contains(rcLog, want) {
|
||||
t.Fatalf("rc-service log missing %q:\n%s", want, rcLog)
|
||||
}
|
||||
}
|
||||
rcuLog := fx.log(t, fx.rcuLog)
|
||||
if !strings.Contains(rcuLog, "add rsmon-worker default") {
|
||||
t.Fatalf("rc-update add not issued:\n%s", rcuLog)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(fx.rcDir, "rsmon-worker.enabled")); err != nil {
|
||||
t.Fatalf("service not enabled in the runlevel: %v", err)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(fx.params.DataDir, "worker.pid")); !os.IsNotExist(err) {
|
||||
t.Fatalf("openrc activation created a pid file; the health loop must use rc-service status")
|
||||
}
|
||||
fx.assertNoLeftovers(t)
|
||||
}
|
||||
|
||||
// TestActivateShellSystemdRollbackFreshDisablesUnit proves a failed
|
||||
// activation on a fresh host disables the newly-enabled unit and removes
|
||||
// the unit file (no dangling enablement).
|
||||
func TestActivateShellSystemdRollbackFreshDisablesUnit(t *testing.T) {
|
||||
fx := newInitShellFixture(t, "systemd")
|
||||
if err := os.WriteFile(filepath.Join(fx.sysdDir, "fail-restart"), []byte(""), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
out, err := fx.run(t)
|
||||
if err == nil {
|
||||
t.Fatal("activation succeeded despite restart failure")
|
||||
}
|
||||
if !strings.Contains(out, "start failure") {
|
||||
t.Fatalf("no start-failure classification:\n%s", out)
|
||||
}
|
||||
log := fx.log(t, fx.sysctlLog)
|
||||
if !strings.Contains(log, "disable rsmon-worker.service") {
|
||||
t.Fatalf("fresh rollback did not disable the unit:\n%s", log)
|
||||
}
|
||||
if _, err := os.Stat(fx.params.UnitFile); !os.IsNotExist(err) {
|
||||
t.Fatal("unit file left behind after fresh rollback")
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(fx.sysdDir, "rsmon-worker.service.enabled")); !os.IsNotExist(err) {
|
||||
t.Fatal("unit still enabled after fresh rollback")
|
||||
}
|
||||
fx.assertNoLeftovers(t)
|
||||
}
|
||||
|
||||
// TestActivateShellSystemdRollbackRestoresEnableState proves a failed
|
||||
// activation on a host with a prior enabled unit restores the unit
|
||||
// byte-for-byte and re-applies the prior enabled state.
|
||||
func TestActivateShellSystemdRollbackRestoresEnableState(t *testing.T) {
|
||||
fx := newInitShellFixture(t, "systemd")
|
||||
if err := os.WriteFile(fx.params.UnitFile, []byte("PRIOR-UNIT-CONTENT"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(fx.sysdDir, "rsmon-worker.service.enabled"), []byte(""), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(fx.sysdDir, "fail-restart"), []byte(""), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
out, err := fx.run(t)
|
||||
if err == nil {
|
||||
t.Fatal("activation succeeded despite restart failure")
|
||||
}
|
||||
if !strings.Contains(out, "start failure") {
|
||||
t.Fatalf("no start-failure classification:\n%s", out)
|
||||
}
|
||||
unit, err := os.ReadFile(fx.params.UnitFile)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if string(unit) != "PRIOR-UNIT-CONTENT" {
|
||||
t.Fatalf("prior unit not restored: %q", unit)
|
||||
}
|
||||
log := fx.log(t, fx.sysctlLog)
|
||||
if !strings.Contains(log, "enable rsmon-worker.service") {
|
||||
t.Fatalf("prior enabled state not re-applied:\n%s", log)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(fx.sysdDir, "rsmon-worker.service.enabled")); err != nil {
|
||||
t.Fatalf("unit not re-enabled after rollback: %v", err)
|
||||
}
|
||||
fx.assertNoLeftovers(t)
|
||||
}
|
||||
|
||||
// TestActivateShellStaleLockBroken proves a leftover lock from a killed
|
||||
// run (dead pid) is broken automatically and the activation proceeds.
|
||||
func TestActivateShellStaleLockBroken(t *testing.T) {
|
||||
fx := newInitShellFixture(t, "systemd")
|
||||
lockDir := filepath.Join(fx.params.DataDir, ".rsmon-activate.lock")
|
||||
if err := os.MkdirAll(lockDir, 0o700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(lockDir, "pid"), []byte("999999\n"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
out, err := fx.run(t)
|
||||
if err != nil {
|
||||
t.Fatalf("activation failed with a stale lock: %v\n%s", err, out)
|
||||
}
|
||||
if !strings.Contains(out, "rsmon-worker activated") {
|
||||
t.Fatalf("no activation success line:\n%s", out)
|
||||
}
|
||||
if _, err := os.Stat(lockDir); !os.IsNotExist(err) {
|
||||
t.Fatal("lock not released after activation")
|
||||
}
|
||||
}
|
||||
|
||||
// TestActivateShellRecoveryFromInterruptedBackup proves the interrupted-run
|
||||
// recovery: a leftover backup marker from a killed activation is restored
|
||||
// before the fresh run, so a later validation failure still leaves the
|
||||
// recovered prior install in place.
|
||||
func TestActivateShellRecoveryFromInterruptedBackup(t *testing.T) {
|
||||
fx := newInitShellFixture(t, "none")
|
||||
backupDir := filepath.Join(fx.params.DataDir, ".rsmon-backup")
|
||||
if err := os.MkdirAll(backupDir, 0o700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(backupDir, "binary"), []byte("RECOVERED-BINARY"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(backupDir, "worker.env"), []byte("RSMON_URL=https://rsmon.ru\nRSMON_TOKEN=test\n"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(backupDir, ".marker"), []byte("0"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Corrupt the stage so the fresh activation fails at validation (before
|
||||
// the fresh snapshot), proving the recovered files were already in place.
|
||||
if err := os.WriteFile(fx.params.Stage, []byte("not a script"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
out, err := fx.run(t)
|
||||
if err == nil {
|
||||
t.Fatal("activation succeeded with a corrupt stage")
|
||||
}
|
||||
if !strings.Contains(out, "recovering interrupted activation") {
|
||||
t.Fatalf("recovery did not run:\n%s", out)
|
||||
}
|
||||
b, err := os.ReadFile(fx.params.Binary)
|
||||
if err != nil {
|
||||
t.Fatalf("recovered binary not restored: %v", err)
|
||||
}
|
||||
if string(b) != "RECOVERED-BINARY" {
|
||||
t.Fatalf("recovered binary content = %q", b)
|
||||
}
|
||||
if _, err := os.Stat(backupDir); !os.IsNotExist(err) {
|
||||
t.Fatal("backup not consumed by recovery")
|
||||
}
|
||||
}
|
||||
409
internal/installer/sourceactivate_test.go
Обычный файл
409
internal/installer/sourceactivate_test.go
Обычный файл
@@ -0,0 +1,409 @@
|
||||
package installer
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"rocketgit.ru/rsmon/worker/internal/sshinstall"
|
||||
)
|
||||
|
||||
func TestActivateScriptMarkers(t *testing.T) {
|
||||
script := activateScript(activateParams{
|
||||
Stage: "/opt/rsmon-worker-src/rsmon-worker",
|
||||
Binary: "/usr/local/bin/rsmon-worker",
|
||||
ConfigDir: "/etc/rsmon-worker",
|
||||
EnvFile: "/etc/rsmon-worker/worker.env",
|
||||
EnvTmp: "/tmp/rsmon-worker-abc.env",
|
||||
DataDir: "/var/lib/rsmon-worker",
|
||||
UnitTmp: "/tmp/rsmon-worker-abc.service",
|
||||
UnitFile: "/etc/systemd/system/rsmon-worker.service",
|
||||
UnitMode: "0644",
|
||||
UnitName: "rsmon-worker.service",
|
||||
RCName: "rsmon-worker",
|
||||
Supervisor: "none",
|
||||
NoStart: "0",
|
||||
})
|
||||
for _, want := range []string{
|
||||
"set -eu",
|
||||
// staged binary validated before any state is touched
|
||||
"\"$stage\" --version >/dev/null",
|
||||
// snapshot + rollback
|
||||
"backup_dir=\"$data_dir/.rsmon-backup\"",
|
||||
"rm -rf \"$backup_dir\"",
|
||||
"cp -p \"$binary\" \"$backup_dir/binary\"",
|
||||
"cp -p \"$env_file\" \"$backup_dir/worker.env\"",
|
||||
"printf '%s\\n' \"$unit_enabled\" > \"$backup_dir/.marker\"",
|
||||
"trap rollback EXIT HUP INT TERM",
|
||||
"restoring the prior install",
|
||||
// atomic installs with the right perms
|
||||
"install -m 0755 \"$stage\" \"$binary.new\"",
|
||||
"mv -f \"$binary.new\" \"$binary\"",
|
||||
"install -m 0600 \"$env_tmp\" \"$env_file.new\"",
|
||||
"mv -f \"$env_file.new\" \"$env_file\"",
|
||||
"chmod 0750 \"$config_dir\"",
|
||||
"mkdir -p \"$data_dir\" \"$data_dir/webapp\"",
|
||||
"install -m \"$unit_mode\" \"$unit_tmp\" \"$unit_file.new\"",
|
||||
"mv -f \"$unit_file.new\" \"$unit_file\"",
|
||||
// supervisor + pid + health verification
|
||||
"pid_file=\"$data_dir/worker.pid\"",
|
||||
"env -i PATH=\"/usr/bin:/bin:/sbin:/usr/sbin\"",
|
||||
"RSMON_WORKER_ENV_FILE=\"$env_file\"",
|
||||
". \"$RSMON_WORKER_ENV_FILE\"",
|
||||
"exec \"$@\"",
|
||||
"nohup \"$1\" >>\"$2\" 2>&1 & echo $! > \"$3\"",
|
||||
"run_env \"$binary\" liveness >/dev/null 2>&1",
|
||||
"while [ \"$i\" -lt 30 ]; do",
|
||||
"rsmon-worker activated:",
|
||||
// activation lock + interrupted-run recovery
|
||||
"lock_dir=\"$data_dir/.rsmon-activate.lock\"",
|
||||
"acquire_lock",
|
||||
"another rsmon-worker activation is in progress",
|
||||
"recovering interrupted activation from $backup_dir",
|
||||
".marker",
|
||||
// pid belongs to the expected binary before kill
|
||||
"readlink \"/proc/$1/exe\"",
|
||||
"\"$binary (deleted)\"",
|
||||
"process_is_worker \"$pid\"",
|
||||
// rollback preserves metadata and enable state
|
||||
"cp -p \"$backup_dir/binary\" \"$binary\"",
|
||||
"cp -p \"$backup_dir/worker.env\" \"$env_file\"",
|
||||
"set_unit_enabled \"$unit_enabled\"",
|
||||
"systemctl is-enabled",
|
||||
// temp/backup cleanup on success
|
||||
"rm -rf \"$backup_dir\"",
|
||||
"release_lock",
|
||||
"trap - EXIT HUP INT TERM",
|
||||
} {
|
||||
if !strings.Contains(script, want) {
|
||||
t.Fatalf("activate script missing %q:\n%s", want, script)
|
||||
}
|
||||
}
|
||||
// The rollback must be armed only after the snapshot so a corrupt
|
||||
// staging binary (validated first) never triggers a destructive
|
||||
// rollback of the prior install.
|
||||
if !strings.Contains(script, "\"$stage\" --version") || !strings.Contains(script, "trap rollback EXIT HUP INT TERM") {
|
||||
t.Fatalf("activate script must validate the staged binary before arming rollback:\n%s", script)
|
||||
}
|
||||
stageCheck := strings.Index(script, "\"$stage\" --version")
|
||||
trapIdx := strings.Index(script, "trap rollback EXIT HUP INT TERM")
|
||||
if stageCheck < 0 || trapIdx < stageCheck {
|
||||
t.Fatalf("staged-binary validation must precede the rollback trap:\n%s", script)
|
||||
}
|
||||
}
|
||||
|
||||
func TestActivateScriptSystemdSupervisor(t *testing.T) {
|
||||
script := activateScript(activateParams{
|
||||
Stage: "/opt/rsmon-worker-src/rsmon-worker",
|
||||
Binary: "/usr/local/bin/rsmon-worker",
|
||||
ConfigDir: "/etc/rsmon-worker",
|
||||
EnvFile: "/etc/rsmon-worker/worker.env",
|
||||
EnvTmp: "/tmp/e.env",
|
||||
DataDir: "/var/lib/rsmon-worker",
|
||||
UnitTmp: "/tmp/u.service",
|
||||
UnitFile: "/etc/systemd/system/rsmon-worker.service",
|
||||
UnitMode: "0644",
|
||||
UnitName: "rsmon-worker.service",
|
||||
RCName: "rsmon-worker",
|
||||
Supervisor: "systemd",
|
||||
NoStart: "0",
|
||||
})
|
||||
for _, want := range []string{
|
||||
"supervisor=systemd",
|
||||
"systemctl daemon-reload",
|
||||
"systemctl restart \"$unit_name\"",
|
||||
"systemctl is-active --quiet \"$unit_name\"",
|
||||
"systemctl enable \"$unit_name\"",
|
||||
"svc_active()",
|
||||
"start failure: systemctl restart $unit_name failed",
|
||||
"start failure: worker service is not active",
|
||||
} {
|
||||
if !strings.Contains(script, want) {
|
||||
t.Fatalf("systemd supervisor missing %q:\n%s", want, script)
|
||||
}
|
||||
}
|
||||
// The systemd health loop must use `systemctl is-active`, never the
|
||||
// embedded supervisor's pid file.
|
||||
if !strings.Contains(script, "systemd) systemctl is-active --quiet \"$unit_name\"") {
|
||||
t.Fatalf("systemd health loop must use systemctl is-active:\n%s", script)
|
||||
}
|
||||
}
|
||||
|
||||
func TestActivateScriptOpenRCSupervisor(t *testing.T) {
|
||||
script := activateScript(activateParams{
|
||||
Stage: "/opt/rsmon-worker-src/rsmon-worker",
|
||||
Binary: "/usr/local/bin/rsmon-worker",
|
||||
ConfigDir: "/etc/rsmon-worker",
|
||||
EnvFile: "/etc/rsmon-worker/worker.env",
|
||||
EnvTmp: "/tmp/e.env",
|
||||
DataDir: "/var/lib/rsmon-worker",
|
||||
UnitTmp: "/tmp/u",
|
||||
UnitFile: "/etc/init.d/rsmon-worker",
|
||||
UnitMode: "0755",
|
||||
UnitName: "rsmon-worker.service",
|
||||
RCName: "rsmon-worker",
|
||||
Supervisor: "openrc",
|
||||
NoStart: "0",
|
||||
})
|
||||
for _, want := range []string{
|
||||
"rc-service \"$rc_name\" restart",
|
||||
"rc-service \"$rc_name\" status",
|
||||
"rc-update add \"$rc_name\" default",
|
||||
"start failure: rc-service restart $rc_name failed",
|
||||
"rc-update del \"$rc_name\" default",
|
||||
} {
|
||||
if !strings.Contains(script, want) {
|
||||
t.Fatalf("openrc supervisor missing %q:\n%s", want, script)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestActivateScriptNoStartSkipsRestart(t *testing.T) {
|
||||
script := activateScript(activateParams{
|
||||
Stage: "/opt/rsmon-worker-src/rsmon-worker", Binary: "/usr/local/bin/rsmon-worker",
|
||||
ConfigDir: "/etc/rsmon-worker", EnvFile: "/etc/rsmon-worker/worker.env", EnvTmp: "/tmp/e.env",
|
||||
DataDir: "/var/lib/rsmon-worker", UnitTmp: "", UnitFile: "", UnitMode: "",
|
||||
UnitName: "rsmon-worker.service", RCName: "rsmon-worker", Supervisor: "none", NoStart: "1",
|
||||
})
|
||||
if !strings.Contains(script, "[ \"$no_start\" -ne 1 ]") {
|
||||
t.Fatalf("no-start gate missing:\n%s", script)
|
||||
}
|
||||
if !strings.Contains(script, "started=no") {
|
||||
t.Fatalf("no-start summary missing:\n%s", script)
|
||||
}
|
||||
}
|
||||
|
||||
func TestActivateScriptNeverContainsSecrets(t *testing.T) {
|
||||
const secret = "super-secret-token-value"
|
||||
params := activateParams{
|
||||
Stage: "/opt/rsmon-worker-src/rsmon-worker", Binary: "/usr/local/bin/rsmon-worker",
|
||||
ConfigDir: "/etc/rsmon-worker", EnvFile: "/etc/rsmon-worker/worker.env", EnvTmp: "/tmp/e.env",
|
||||
DataDir: "/var/lib/rsmon-worker", UnitTmp: "/tmp/u", UnitFile: "/etc/systemd/system/rsmon-worker.service",
|
||||
UnitMode: "0644", UnitName: "rsmon-worker.service", RCName: "rsmon-worker",
|
||||
Supervisor: "systemd", NoStart: "0",
|
||||
}
|
||||
script := activateScript(params)
|
||||
if strings.Contains(script, secret) {
|
||||
t.Fatalf("activate script contains a secret:\n%s", script)
|
||||
}
|
||||
// The env file is referenced by path; its contents (the secrets) are
|
||||
// only sourced at runtime and never echoed.
|
||||
for _, want := range []string{"RSMON_TOKEN=", secret} {
|
||||
if strings.Contains(script, want) {
|
||||
t.Fatalf("activate script must not embed env contents (%q):\n%s", want, script)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenRCInit(t *testing.T) {
|
||||
unit := openrcInitFor(resolvePaths(""))
|
||||
for _, want := range []string{
|
||||
"#!/sbin/openrc-run",
|
||||
"name=rsmon-worker",
|
||||
"description=RSMon distributed monitoring worker",
|
||||
"command=/usr/local/bin/rsmon-worker",
|
||||
"command_background=true",
|
||||
"pidfile=/var/lib/rsmon-worker/worker.pid",
|
||||
"output_log=/var/lib/rsmon-worker/worker.log",
|
||||
"need net",
|
||||
". /etc/rsmon-worker/worker.env",
|
||||
"set -a",
|
||||
"set +a",
|
||||
"RSMON_WEBAPP_DATA_DIR=/var/lib/rsmon-worker/webapp",
|
||||
} {
|
||||
if !strings.Contains(unit, want) {
|
||||
t.Fatalf("openrc init missing %q:\n%s", want, unit)
|
||||
}
|
||||
}
|
||||
named := openrcInitFor(resolvePaths("edge"))
|
||||
for _, want := range []string{
|
||||
"name=rsmon-worker-edge", "command=/usr/local/bin/rsmon-worker-edge",
|
||||
"pidfile=/var/lib/rsmon-worker-edge/worker.pid", "/etc/rsmon-worker-edge/worker.env", "(edge)",
|
||||
} {
|
||||
if !strings.Contains(named, want) {
|
||||
t.Fatalf("named openrc init missing %q:\n%s", want, named)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunningInitScript(t *testing.T) {
|
||||
for _, want := range []string{"set -eu", "/run/systemd/system", "systemctl", "/run/openrc/softlevel", "rc-service", "echo none"} {
|
||||
if !strings.Contains(runningInitScript, want) {
|
||||
t.Fatalf("running-init script missing %q:\n%s", want, runningInitScript)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeSupervisor(t *testing.T) {
|
||||
for _, in := range []string{"systemd", "openrc", "systemd\n", " openrc "} {
|
||||
want := strings.TrimSpace(in)
|
||||
if got := normalizeSupervisor(in); got != want {
|
||||
t.Fatalf("normalizeSupervisor(%q) = %q, want %q", in, got, want)
|
||||
}
|
||||
}
|
||||
for _, in := range []string{"", "none", "sysvinit", " "} {
|
||||
if got := normalizeSupervisor(in); got != "none" {
|
||||
t.Fatalf("normalizeSupervisor(%q) = %q, want none", in, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestUnitContent(t *testing.T) {
|
||||
p := resolvePaths("")
|
||||
systemd, file, mode := unitContent(sshinstall.InitSystemd, p)
|
||||
if systemd == "" || file != "/etc/systemd/system/rsmon-worker.service" || mode != "0644" {
|
||||
t.Fatalf("systemd unit content = %q, %q, %q", systemd, file, mode)
|
||||
}
|
||||
if !strings.Contains(systemd, "ExecStart=/usr/local/bin/rsmon-worker\n") {
|
||||
t.Fatalf("systemd unit not the classic hardened unit:\n%s", systemd)
|
||||
}
|
||||
openrc, file, mode := unitContent(sshinstall.InitOpenRC, p)
|
||||
if openrc == "" || file != "/etc/init.d/rsmon-worker" || mode != "0755" {
|
||||
t.Fatalf("openrc unit content = %q, %q, %q", openrc, file, mode)
|
||||
}
|
||||
if content, file, mode := unitContent(sshinstall.InitUnknown, p); content != "" || file != "" || mode != "" {
|
||||
t.Fatalf("unknown-init unit content = %q, %q, %q, want empty (no-service gate)", content, file, mode)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRenderEnvActivation(t *testing.T) {
|
||||
t.Setenv("RSMON_URL", "")
|
||||
t.Setenv("RSMON_TOKEN", "")
|
||||
t.Setenv("WORKER_HOST", "")
|
||||
t.Setenv("WORKER_PORT", "")
|
||||
t.Setenv("WORKER_LOGIN", "")
|
||||
t.Setenv("WORKER_PASSWORD", "")
|
||||
|
||||
data, err := (ActivationOptions{Activate: true, URL: "https://rsmon.ru", Token: "secret"}).renderEnv()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, want := range []string{"RSMON_URL=https://rsmon.ru\n", "RSMON_TOKEN=secret\n", "WORKER_HOST=127.0.0.1\n", "WORKER_PORT=27401\n"} {
|
||||
if !strings.Contains(string(data), want) {
|
||||
t.Fatalf("rendered env missing %q:\n%s", want, data)
|
||||
}
|
||||
}
|
||||
|
||||
// PUBLIC_URL canonicalization is reused from the classic installer.
|
||||
data, err = (ActivationOptions{
|
||||
Activate: true, URL: "https://rsmon.ru", Token: "secret",
|
||||
PublicURL: "https://worker.example.com",
|
||||
}).renderEnv()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.Contains(string(data), "PUBLIC_URL=https://worker.example.com\n") {
|
||||
t.Fatalf("rendered env missing PUBLIC_URL:\n%s", data)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRenderEnvActivationRejectsMixedBasicAuth(t *testing.T) {
|
||||
t.Setenv("RSMON_TOKEN", "")
|
||||
if _, err := (ActivationOptions{
|
||||
Activate: true, URL: "https://rsmon.ru", Token: "secret",
|
||||
Login: "admin",
|
||||
}).renderEnv(); err == nil {
|
||||
t.Fatal("login-only basic auth accepted")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRenderEnvActivationRequiresToken(t *testing.T) {
|
||||
if _, err := (ActivationOptions{Activate: true, URL: "https://rsmon.ru"}).renderEnv(); err == nil {
|
||||
t.Fatal("activation without a token accepted")
|
||||
}
|
||||
if _, err := (ActivationOptions{Activate: true, Token: "x", URL: "not-a-url"}).renderEnv(); err == nil {
|
||||
t.Fatal("activation with a malformed URL accepted")
|
||||
}
|
||||
if _, err := (ActivationOptions{Activate: true, Token: "x", URL: "https://rsmon.ru", Name: "Bad_Name"}).renderEnv(); err == nil {
|
||||
t.Fatal("activation with an invalid instance name accepted")
|
||||
}
|
||||
if _, err := (ActivationOptions{Activate: true, Token: "x", URL: "https://rsmon.ru", Name: "edge"}).renderEnv(); err == nil {
|
||||
t.Fatal("named activation without WORKER_PORT accepted")
|
||||
}
|
||||
}
|
||||
|
||||
// TestRenderEnvActivationReadsEnvFileOnce proves the env file is read a
|
||||
// single time and validated from the in-memory bytes: swapping the file
|
||||
// after the read cannot smuggle a different value into the render, and a
|
||||
// malformed file fails on the read bytes.
|
||||
func TestRenderEnvActivationReadsEnvFileOnce(t *testing.T) {
|
||||
t.Setenv("RSMON_TOKEN", "")
|
||||
path := filepath.Join(t.TempDir(), "worker.env")
|
||||
if err := os.WriteFile(path, []byte("RSMON_URL=https://rsmon.ru\nRSMON_TOKEN=file-token\nWORKER_PORT=28888\n"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
data, err := (ActivationOptions{Activate: true, EnvFile: path}).renderEnv()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.Contains(string(data), "RSMON_TOKEN=file-token\n") || !strings.Contains(string(data), "WORKER_PORT=28888\n") {
|
||||
t.Fatalf("env file values not rendered:\n%s", data)
|
||||
}
|
||||
// A malformed file must be rejected from the same read.
|
||||
if err := os.WriteFile(path, []byte("RSMON_URL=https://rsmon.ru\nRSMON_TOKEN=has space\n"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := (ActivationOptions{Activate: true, EnvFile: path}).renderEnv(); err == nil {
|
||||
t.Fatal("malformed env file accepted")
|
||||
}
|
||||
}
|
||||
|
||||
// TestNormalizeSourceOptionsActivationRendersOnce verifies the rendered
|
||||
// env is computed during normalization and reused, so a later call cannot
|
||||
// re-read a changed env file.
|
||||
func TestNormalizeSourceOptionsActivationRendersOnce(t *testing.T) {
|
||||
t.Setenv("RSMON_TOKEN", "")
|
||||
path := filepath.Join(t.TempDir(), "worker.env")
|
||||
if err := os.WriteFile(path, []byte("RSMON_URL=https://rsmon.ru\nRSMON_TOKEN=once-token\n"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
base := SourceInstallOptions{
|
||||
SSHOptions: SSHOptions{Host: "h", User: "u"},
|
||||
Activation: ActivationOptions{Activate: true, EnvFile: path},
|
||||
}
|
||||
norm, err := normalizeSourceOptions(base)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.Contains(string(norm.activationEnv), "RSMON_TOKEN=once-token\n") {
|
||||
t.Fatalf("activation env not cached during normalization: %q", norm.activationEnv)
|
||||
}
|
||||
// Even after the file changes, the cached render is authoritative.
|
||||
if err := os.WriteFile(path, []byte("RSMON_URL=https://evil.test\nRSMON_TOKEN=evil\n"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if strings.Contains(string(norm.activationEnv), "evil") {
|
||||
t.Fatalf("changed env file leaked into the cached render: %q", norm.activationEnv)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeSourceOptionsActivation(t *testing.T) {
|
||||
t.Setenv("RSMON_TOKEN", "")
|
||||
base := SourceInstallOptions{SSHOptions: SSHOptions{Host: "h", User: "u"}}
|
||||
if _, err := normalizeSourceOptions(base); err != nil {
|
||||
t.Fatalf("staging-only options must stay valid: %v", err)
|
||||
}
|
||||
act := base
|
||||
act.Activation = ActivationOptions{Activate: true, URL: "https://rsmon.ru"}
|
||||
if _, err := normalizeSourceOptions(act); err == nil || !strings.Contains(err.Error(), "RSMON_TOKEN") {
|
||||
t.Fatalf("activation without a token error = %v", err)
|
||||
}
|
||||
ok := base
|
||||
ok.Activation = ActivationOptions{Activate: true, URL: "https://rsmon.ru", Token: "secret"}
|
||||
if _, err := normalizeSourceOptions(ok); err != nil {
|
||||
t.Fatalf("valid activation rejected: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateRenderedEnv(t *testing.T) {
|
||||
if err := validateRenderedEnv([]byte("RSMON_URL=https://rsmon.ru\nRSMON_TOKEN=secret\n")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := validateRenderedEnv([]byte("RSMON_TOKEN=secret\n")); err == nil {
|
||||
t.Fatal("env without RSMON_URL accepted")
|
||||
}
|
||||
if err := validateRenderedEnv([]byte("RSMON_URL=https://rsmon.ru\nRSMON_TOKEN=secret value\n")); err == nil {
|
||||
t.Fatal("env with whitespace accepted")
|
||||
}
|
||||
}
|
||||
603
internal/installer/sourceinstall.go
Обычный файл
603
internal/installer/sourceinstall.go
Обычный файл
@@ -0,0 +1,603 @@
|
||||
package installer
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"golang.org/x/crypto/ssh"
|
||||
|
||||
"rocketgit.ru/rsmon/worker/internal/sshinstall"
|
||||
)
|
||||
|
||||
// SourceInstallOptions drives the remote source installation (work
|
||||
// package 3 of docs/source-installation.md). It reuses the deploy
|
||||
// command's SSHOptions for authentication and host-key verification and
|
||||
// adds the source-build knobs. It never carries a worker token or
|
||||
// control-plane credential: the built worker is staged, not configured
|
||||
// or started, so no secret is ever sent to the remote host.
|
||||
type SourceInstallOptions struct {
|
||||
SSHOptions
|
||||
|
||||
// Repo is the public worker repository to clone or update. Empty
|
||||
// uses the sshinstall default. Only https URLs without userinfo are
|
||||
// accepted.
|
||||
Repo string
|
||||
// Branch pins the branch to build. Empty resolves the remote's
|
||||
// default branch (the public repo currently publishes "master");
|
||||
// the resolved branch and commit are recorded in the build dir.
|
||||
Branch string
|
||||
// GoVersion defaults to the pinned sshinstall toolchain (1.26.0).
|
||||
// Non-default versions have no baked checksum yet and are rejected.
|
||||
GoVersion string
|
||||
// GoArch optionally pins the Go download archive suffix (e.g.
|
||||
// "amd64"); empty derives it from the remote `uname -m`.
|
||||
GoArch string
|
||||
// BuildDir is the remote clone/build directory.
|
||||
BuildDir string
|
||||
// GoModuleProxy overrides GOPROXY for the remote build.
|
||||
GoModuleProxy string
|
||||
// ToolchainDir is where the verified Go toolchain is installed.
|
||||
// It must be an absolute path ending in /go (default
|
||||
// /usr/local/go). Replacement is atomic: the new toolchain is
|
||||
// downloaded, verified, and staged before the prior one is moved
|
||||
// aside, and the prior one is restored if the swap fails.
|
||||
ToolchainDir string
|
||||
// StageBinary is where the built worker binary is written. It must
|
||||
// be absolute and defaults to <BuildDir>/rsmon-worker. The running
|
||||
// service and its config are NOT touched until Activation runs.
|
||||
StageBinary string
|
||||
// SessionTimeout bounds each remote command. 0 uses the default
|
||||
// (30 minutes); the build step can legitimately run for minutes.
|
||||
SessionTimeout time.Duration
|
||||
// Activation drives work package 4: after the staging build, the
|
||||
// staged binary, validated environment, data dir, and the detected
|
||||
// init's service definition are installed atomically and the worker
|
||||
// is started and verified (process + /healthz). Any failure rolls
|
||||
// back to the prior working install. Empty keeps SourceInstall at the
|
||||
// staging boundary and touches no service configuration.
|
||||
Activation ActivationOptions
|
||||
|
||||
// activationEnv is the once-rendered activation environment, computed
|
||||
// during option normalization so the env file is read and rendered
|
||||
// exactly once per run (no TOCTOU between preflight and activation).
|
||||
activationEnv []byte
|
||||
}
|
||||
|
||||
// SourceInstallResult is what a source installation resolved to. The
|
||||
// resolved branch and commit are recorded on the remote host in
|
||||
// RecordFile, and the built binary is left at StageBinary for the next
|
||||
// (service-activation) work package to install atomically.
|
||||
type SourceInstallResult struct {
|
||||
Detection sshinstall.Detection
|
||||
Plan sshinstall.SourcePlan
|
||||
GoArch string // resolved archive suffix, e.g. "linux-amd64"
|
||||
ToolchainDir string
|
||||
ResolvedBranch string
|
||||
ResolvedCommit string
|
||||
RecordFile string
|
||||
StageBinary string
|
||||
// Activation is set when the staged binary was atomically installed
|
||||
// and the worker started/verified. It records the installed layout
|
||||
// and the supervisor used. Nil when Activation was not requested.
|
||||
Activation *ActivationResult
|
||||
}
|
||||
|
||||
// defaultToolchainDir is the standard Go installation prefix.
|
||||
const defaultToolchainDir = "/usr/local/go"
|
||||
|
||||
// defaultSessionTimeout bounds each remote command when the operator
|
||||
// does not configure one. The staging build and first-time module
|
||||
// downloads can run for minutes, so this is generous.
|
||||
const defaultSessionTimeout = 30 * time.Minute
|
||||
|
||||
// commitRecordName is the file (inside BuildDir) that records the
|
||||
// resolved branch and commit the build was produced from.
|
||||
const commitRecordName = "rsmon-worker.commit"
|
||||
|
||||
var (
|
||||
commitPattern = regexp.MustCompile(`^[0-9a-f]{40}$`)
|
||||
branchNamePattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._/-]*$`)
|
||||
)
|
||||
|
||||
// sourceExecutor bundles the SSH client, privilege options, and per
|
||||
// command timeout used by every remote source-install step.
|
||||
type sourceExecutor struct {
|
||||
client *ssh.Client
|
||||
ssh SSHOptions
|
||||
timeout time.Duration
|
||||
}
|
||||
|
||||
// runPrivileged executes a command through the SSHOptions privilege path
|
||||
// (root, passwordless sudo, or sudo -S) and returns bounded stdout.
|
||||
func (e *sourceExecutor) runPrivileged(command string) ([]byte, error) {
|
||||
cmd, stdin := sudoWrap(e.ssh.User, e.ssh.SudoPassword, command)
|
||||
return runRemoteOutput(e.client, cmd, stdin, e.timeout)
|
||||
}
|
||||
|
||||
// runPlain executes a command as the SSH user and returns bounded
|
||||
// stdout.
|
||||
func (e *sourceExecutor) runPlain(command string) ([]byte, error) {
|
||||
return runRemoteOutput(e.client, command, nil, e.timeout)
|
||||
}
|
||||
|
||||
// fileProber builds the sshinstall.FileProber used for init-system
|
||||
// detection over a live SSH session.
|
||||
func (e *sourceExecutor) fileProber() sshinstall.FileProber {
|
||||
return func(paths ...string) map[string]bool {
|
||||
quoted := make([]string, len(paths))
|
||||
for i, p := range paths {
|
||||
quoted[i] = shellQuote(p)
|
||||
}
|
||||
expr := "for p in " + strings.Join(quoted, " ") + "; do [ -e \"$p\" ] && printf '%s\\n' \"$p\"; done; true"
|
||||
out, err := e.runPlain(expr)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
present := make(map[string]bool, len(paths))
|
||||
for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") {
|
||||
if line = strings.TrimSpace(line); line != "" {
|
||||
present[line] = true
|
||||
}
|
||||
}
|
||||
return present
|
||||
}
|
||||
}
|
||||
|
||||
// SourceInstall executes the source-install flow over SSH: it reuses the
|
||||
// deploy command's SSH authentication and host-key verification, detects
|
||||
// the remote host, plans the pinned toolchain and package prerequisites,
|
||||
// then installs packages, downloads and verifies the Go toolchain,
|
||||
// clones/updates the public repository (verifying an existing checkout's
|
||||
// origin matches the configured repository), checks out the resolved
|
||||
// branch, builds the worker to a staging path, and only then records the
|
||||
// resolved branch and commit.
|
||||
//
|
||||
// When Activation.Activate is set, the staged binary is then atomically
|
||||
// installed together with the validated environment, data directory, and
|
||||
// the detected init's service definition, and the worker is started and
|
||||
// verified (process + /healthz); any activation/start/health failure
|
||||
// rolls back to the prior working install. Without activation the
|
||||
// running service, its configuration, and its data directory are
|
||||
// deliberately untouched (the staging boundary of work package 3).
|
||||
//
|
||||
// Every remote step runs with the same privilege path as `deploy` (root,
|
||||
// passwordless sudo, or sudo -S), every interpolated value is
|
||||
// single-quoted, every step script fails closed (`set -eu` or explicit
|
||||
// `&&`/retry), and every failure returns a bounded, actionable error.
|
||||
// Each remote command is capped by SessionTimeout and its stdout is
|
||||
// size-bounded.
|
||||
func SourceInstall(opts SourceInstallOptions) (*SourceInstallResult, error) {
|
||||
opts, err := normalizeSourceOptions(opts)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
auth, err := sshAuth(opts.SSHOptions)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
hostKey, err := hostKeyCallback(opts.SSHOptions)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
client, err := ssh.Dial("tcp", net.JoinHostPort(opts.Host, strconv.Itoa(opts.Port)), &ssh.ClientConfig{
|
||||
User: opts.User,
|
||||
Auth: auth,
|
||||
HostKeyCallback: hostKey,
|
||||
Timeout: 15 * time.Second,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("connect to %s: %w", opts.Host, err)
|
||||
}
|
||||
defer client.Close() //nolint:errcheck
|
||||
|
||||
executor := &sourceExecutor{client: client, ssh: opts.SSHOptions, timeout: opts.SessionTimeout}
|
||||
|
||||
osRelease, err := executor.runPlain("cat /etc/os-release")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read remote /etc/os-release: %w", err)
|
||||
}
|
||||
uname, err := executor.runPlain("uname -m")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read remote machine architecture: %w", err)
|
||||
}
|
||||
detection := sshinstall.Detect(string(osRelease), executor.fileProber())
|
||||
plan, err := sshinstall.PlanSource(detection, sshinstall.SourceOptions{
|
||||
Repo: opts.Repo,
|
||||
Branch: opts.Branch,
|
||||
GoVersion: opts.GoVersion,
|
||||
GoArch: opts.GoArch,
|
||||
UnameM: strings.TrimSpace(string(uname)),
|
||||
BuildDir: opts.BuildDir,
|
||||
GoModuleProxy: opts.GoModuleProxy,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
stage := opts.StageBinary
|
||||
if stage == "" {
|
||||
stage = filepath.Join(plan.BuildDir, "rsmon-worker")
|
||||
}
|
||||
result := &SourceInstallResult{
|
||||
Detection: detection,
|
||||
Plan: plan,
|
||||
GoArch: plan.Toolchain.Arch,
|
||||
ToolchainDir: opts.ToolchainDir,
|
||||
StageBinary: stage,
|
||||
}
|
||||
|
||||
if len(plan.Packages) > 0 {
|
||||
script := "sh -c " + shellQuote(packageScript(detection.PackageManager, plan.Packages))
|
||||
if _, err := executor.runPrivileged(script); err != nil {
|
||||
return nil, fmt.Errorf("install prerequisites via %s: %w", detection.PackageManager, err)
|
||||
}
|
||||
}
|
||||
|
||||
script := "sh -c " + shellQuote(toolchainScript(plan.Toolchain, opts.ToolchainDir))
|
||||
if _, err := executor.runPrivileged(script); err != nil {
|
||||
return nil, fmt.Errorf("install Go %s toolchain: %w", plan.Toolchain.Version, err)
|
||||
}
|
||||
|
||||
// Clone or update. An existing checkout must point at the configured
|
||||
// repository, or the install fails before fetching or building.
|
||||
script = "sh -c " + shellQuote(cloneUpdateScript(plan.Repo, plan.BuildDir))
|
||||
if _, err := executor.runPrivileged(script); err != nil {
|
||||
return nil, fmt.Errorf("clone/update source repository: %w", err)
|
||||
}
|
||||
|
||||
out, err := executor.runPrivileged("sh -c " + shellQuote(resolveBranchScript(plan.BuildDir)))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("resolve remote default branch: %w", err)
|
||||
}
|
||||
branch, err := parseResolvedBranch(string(out))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if plan.Branch != "" && plan.Branch != branch {
|
||||
ref := "refs/remotes/origin/" + plan.Branch
|
||||
if _, err := executor.runPrivileged("sh -c " + shellQuote(refExistsScript(plan.BuildDir, ref))); err != nil {
|
||||
return nil, fmt.Errorf("branch %q does not exist on the remote repository: %w", plan.Branch, err)
|
||||
}
|
||||
branch = plan.Branch
|
||||
}
|
||||
result.ResolvedBranch = branch
|
||||
|
||||
// Checkout fails closed: a dirty tree or missing branch aborts before
|
||||
// the build, so the previous staging binary is left untouched.
|
||||
out, err = executor.runPrivileged("sh -c " + shellQuote(checkoutScript(plan.BuildDir, branch)))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("check out branch %q: %w", branch, err)
|
||||
}
|
||||
commit, err := parseResolvedCommit(string(out))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("resolve commit: %w", err)
|
||||
}
|
||||
result.ResolvedCommit = commit
|
||||
|
||||
// Build with the repository's own flags (same -ldflags shape the
|
||||
// worker Makefile uses), resolving COMMIT from the checkout and
|
||||
// BUILD_DATE at install time. The binary is built to a temp path,
|
||||
// verified, and atomically swapped into the staging path; a failed
|
||||
// build leaves the previous staging binary in place.
|
||||
buildDate := time.Now().UTC().Format("2006-01-02T15:04:05Z")
|
||||
ldflags := fmt.Sprintf("-s -w -X main.version=dev -X main.commit=%s -X main.buildDate=%s", commit[:12], buildDate)
|
||||
script = "sh -c " + shellQuote(buildScript(opts.ToolchainDir, plan.BuildDir, plan.GoModuleProxy, stage, ldflags))
|
||||
if _, err := executor.runPrivileged(script); err != nil {
|
||||
return nil, fmt.Errorf("build worker binary: %w", err)
|
||||
}
|
||||
|
||||
// The commit record is written only after a successful build, so the
|
||||
// record and the staged binary always correspond to the same commit.
|
||||
result.RecordFile = filepath.Join(plan.BuildDir, commitRecordName)
|
||||
if _, err := executor.runPrivileged("sh -c " + shellQuote(commitRecordScript(plan.BuildDir, branch, commit))); err != nil {
|
||||
return nil, fmt.Errorf("record resolved commit: %w", err)
|
||||
}
|
||||
|
||||
// Work package 4: atomically install the staged build and activate
|
||||
// the worker. On any failure the remote script restores the previous
|
||||
// working install and this step returns a bounded error.
|
||||
if opts.Activation.Activate {
|
||||
if err := executor.activateWorker(opts, result); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// normalizeSourceOptions validates operator input before any remote
|
||||
// connection or mutation. Every value that later reaches a remote shell
|
||||
// is constrained here.
|
||||
func normalizeSourceOptions(o SourceInstallOptions) (SourceInstallOptions, error) {
|
||||
if o.Host == "" || o.User == "" {
|
||||
return o, errors.New("--host and --user are required for source install")
|
||||
}
|
||||
if o.Port == 0 {
|
||||
o.Port = 22
|
||||
}
|
||||
if o.Port < 1 || o.Port > 65535 {
|
||||
return o, errors.New("SSH port must be between 1 and 65535")
|
||||
}
|
||||
if o.Branch != "" && !validBranchName(o.Branch) {
|
||||
return o, fmt.Errorf("invalid branch %q: only A-Za-z0-9, dots, underscores, slashes, and hyphens are allowed", o.Branch)
|
||||
}
|
||||
if o.GoVersion != "" && !sshinstall.ValidGoVersion(o.GoVersion) {
|
||||
return o, fmt.Errorf("invalid Go version %q: only digits, letters, dots, dashes, and underscores are allowed", o.GoVersion)
|
||||
}
|
||||
if o.GoArch != "" && !sshinstall.ValidGoArch(o.GoArch) {
|
||||
return o, fmt.Errorf("invalid Go architecture %q: only letters, digits, dashes, and underscores are allowed", o.GoArch)
|
||||
}
|
||||
if err := sshinstall.ValidateRepoURL(o.Repo); err != nil {
|
||||
return o, fmt.Errorf("invalid repository: %w", err)
|
||||
}
|
||||
if o.ToolchainDir == "" {
|
||||
o.ToolchainDir = defaultToolchainDir
|
||||
}
|
||||
if !strings.HasPrefix(o.ToolchainDir, "/") || filepath.Base(o.ToolchainDir) != "go" {
|
||||
return o, fmt.Errorf("toolchain directory must be an absolute path ending in /go, got %q", o.ToolchainDir)
|
||||
}
|
||||
if o.StageBinary != "" && !strings.HasPrefix(o.StageBinary, "/") {
|
||||
return o, fmt.Errorf("staging binary path must be absolute, got %q", o.StageBinary)
|
||||
}
|
||||
if o.SessionTimeout <= 0 {
|
||||
o.SessionTimeout = defaultSessionTimeout
|
||||
}
|
||||
if o.Activation.Activate {
|
||||
// Render (and validate) the activation environment exactly once
|
||||
// here. The rendered bytes are reused at activation time, so the
|
||||
// env file is read a single time and cannot change between the
|
||||
// preflight and the remote install (env-file TOCTOU).
|
||||
env, err := o.Activation.renderEnv()
|
||||
if err != nil {
|
||||
return o, fmt.Errorf("activation: %w", err)
|
||||
}
|
||||
if err := validateRenderedEnv(env); err != nil {
|
||||
return o, fmt.Errorf("activation: %w", err)
|
||||
}
|
||||
o.activationEnv = env
|
||||
}
|
||||
return o, nil
|
||||
}
|
||||
|
||||
// validBranchName reports whether a branch is a safe git branch name
|
||||
// that can be interpolated into remote commands. The charset check is
|
||||
// the command-injection boundary; the extra rules reject git-invalid or
|
||||
// ambiguous refname patterns.
|
||||
func validBranchName(branch string) bool {
|
||||
if !branchNamePattern.MatchString(branch) {
|
||||
return false
|
||||
}
|
||||
if strings.HasPrefix(branch, "-") || strings.HasPrefix(branch, "/") ||
|
||||
strings.Contains(branch, "..") || strings.Contains(branch, "@{") ||
|
||||
strings.Contains(branch, "//") || strings.HasSuffix(branch, ".") ||
|
||||
strings.HasSuffix(branch, "/") {
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// parseResolvedBranch turns the `git symbolic-ref` output
|
||||
// ("origin/master\n") into the short branch name, validating it so
|
||||
// remote-controlled output can never inject a command.
|
||||
func parseResolvedBranch(raw string) (string, error) {
|
||||
branch := strings.TrimSpace(raw)
|
||||
branch = strings.TrimPrefix(branch, "origin/")
|
||||
if !validBranchName(branch) {
|
||||
return "", fmt.Errorf("remote reported an invalid default branch %q", strings.TrimSpace(raw))
|
||||
}
|
||||
return branch, nil
|
||||
}
|
||||
|
||||
// parseResolvedCommit extracts the 40-hex commit from `git rev-parse
|
||||
// HEAD` output, taking the last whitespace-separated token so unrelated
|
||||
// stdout cannot satisfy the parse.
|
||||
func parseResolvedCommit(raw string) (string, error) {
|
||||
fields := strings.Fields(strings.TrimSpace(raw))
|
||||
if len(fields) == 0 {
|
||||
return "", errors.New("remote reported no commit")
|
||||
}
|
||||
commit := fields[len(fields)-1]
|
||||
if !commitPattern.MatchString(commit) {
|
||||
return "", fmt.Errorf("remote reported an invalid resolved commit %q", commit)
|
||||
}
|
||||
return commit, nil
|
||||
}
|
||||
|
||||
// quoteList renders each item as a single shell-quoted word.
|
||||
func quoteList(items []string) string {
|
||||
quoted := make([]string, len(items))
|
||||
for i, item := range items {
|
||||
quoted[i] = shellQuote(item)
|
||||
}
|
||||
return strings.Join(quoted, " ")
|
||||
}
|
||||
|
||||
// packageScript installs the minimal build prerequisites with the
|
||||
// distro's package manager. It is idempotent on every supported manager
|
||||
// and never installs a C compiler (the worker builds with CGO disabled).
|
||||
func packageScript(pkg sshinstall.PackageManager, pkgs []string) string {
|
||||
quoted := quoteList(pkgs)
|
||||
switch pkg {
|
||||
case sshinstall.PkgApk:
|
||||
return "apk add --no-cache " + quoted
|
||||
case sshinstall.PkgApt:
|
||||
// Ubuntu/Debian need a fresh package index before installing.
|
||||
return "export DEBIAN_FRONTEND=noninteractive\napt-get update\napt-get install -y --no-install-recommends " + quoted
|
||||
case sshinstall.PkgPacman:
|
||||
return "pacman -Sy --noconfirm --needed " + quoted
|
||||
case sshinstall.PkgDnf:
|
||||
return "dnf install -y --setopt=install_weak_deps=False " + quoted
|
||||
case sshinstall.PkgYum:
|
||||
return "yum install -y " + quoted
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
// toolchainScript downloads the pinned Go toolchain, verifies its
|
||||
// published SHA-256 before extraction, stages the extract on the same
|
||||
// filesystem as the target, verifies the staged toolchain reports the
|
||||
// target version, and only then swaps it into place. The prior toolchain
|
||||
// (when present) is moved to a sibling backup first and is restored if
|
||||
// the swap fails, so a failed download/verify/extract/swap always leaves
|
||||
// the prior Go untouched. A present toolchain already reporting the
|
||||
// target version is reused (idempotent rerun). Temp and staging
|
||||
// directories are removed on success and failure.
|
||||
func toolchainScript(tc sshinstall.Toolchain, toolchainDir string) string {
|
||||
parent := filepath.Dir(toolchainDir)
|
||||
want := "go" + tc.Version
|
||||
var b strings.Builder
|
||||
b.WriteString("set -eu\n")
|
||||
b.WriteString("parent=" + shellQuote(parent) + "\n")
|
||||
b.WriteString("toolchain=" + shellQuote(toolchainDir) + "\n")
|
||||
b.WriteString("want=" + shellQuote(want) + "\n")
|
||||
b.WriteString("if [ -x \"$toolchain/bin/go\" ]; then\n")
|
||||
b.WriteString(" have=\"$($toolchain/bin/go version 2>/dev/null | awk '{print $3}')\"\n")
|
||||
b.WriteString(" if [ \"$have\" = \"$want\" ]; then\n")
|
||||
b.WriteString(" echo \"go toolchain already present: $have\"\n")
|
||||
b.WriteString(" exit 0\n")
|
||||
b.WriteString(" fi\n")
|
||||
b.WriteString("fi\n")
|
||||
b.WriteString("work=\"$(mktemp -d /tmp/rsmon-toolchain-XXXXXX)\"\n")
|
||||
b.WriteString("staging=\"$(mktemp -d \"$parent/.go-staging-XXXXXX\")\"\n")
|
||||
b.WriteString("trap 'rm -rf \"$work\" \"$staging\"' EXIT HUP INT TERM\n")
|
||||
b.WriteString("archive=\"$work/go" + tc.Version + "." + tc.Arch + ".tar.gz\"\n")
|
||||
b.WriteString("curl -fsSL --retry 3 --retry-delay 2 -o \"$archive\" " + shellQuote(tc.URL) + "\n")
|
||||
b.WriteString("printf '%s %s\\n' " + shellQuote(tc.SHA256) + " \"$archive\" | sha256sum -c -\n")
|
||||
b.WriteString("tar -C \"$staging\" -xzf \"$archive\"\n")
|
||||
b.WriteString("staged=\"$($staging/go/bin/go version | awk '{print $3}')\"\n")
|
||||
b.WriteString("if [ \"$staged\" != \"$want\" ]; then\n")
|
||||
b.WriteString(" printf 'staged toolchain failed verification: %s\\n' \"$staged\" >&2\n")
|
||||
b.WriteString(" exit 1\n")
|
||||
b.WriteString("fi\n")
|
||||
// Swap atomically on the same filesystem, preserving the prior
|
||||
// toolchain in a sibling backup with rollback on failure.
|
||||
b.WriteString("backup=\"\"\n")
|
||||
b.WriteString("if [ -e \"$toolchain\" ]; then\n")
|
||||
b.WriteString(" backup=\"$parent/.go-backup\"\n")
|
||||
b.WriteString(" rm -rf \"$backup\"\n")
|
||||
b.WriteString(" mv \"$toolchain\" \"$backup\"\n")
|
||||
b.WriteString("fi\n")
|
||||
b.WriteString("if ! mv \"$staging/go\" \"$toolchain\"; then\n")
|
||||
b.WriteString(" if [ -n \"$backup\" ]; then\n")
|
||||
b.WriteString(" mv \"$backup\" \"$toolchain\"\n")
|
||||
b.WriteString(" fi\n")
|
||||
b.WriteString(" exit 1\n")
|
||||
b.WriteString("fi\n")
|
||||
b.WriteString("if [ -n \"$backup\" ]; then\n")
|
||||
b.WriteString(" rm -rf \"$backup\"\n")
|
||||
b.WriteString("fi\n")
|
||||
b.WriteString("\"$toolchain/bin/go\" version\n")
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// cloneUpdateScript clones the repository when missing and otherwise
|
||||
// fetches the latest refs, so a rerun updates in place. Before fetching
|
||||
// an existing checkout, it verifies the configured repository matches
|
||||
// the checkout's `remote.origin.url` exactly, so the installer can never
|
||||
// fetch or build an unconfigured repository. The clone/fetch is retried
|
||||
// up to three times (2s apart) because real repositories can be
|
||||
// transiently unreachable (DNS, TLS, or proxy hiccups). The script fails
|
||||
// closed: any exhausted retry exits non-zero.
|
||||
func cloneUpdateScript(repo, buildDir string) string {
|
||||
return "set -u\n" +
|
||||
"repo=" + shellQuote(repo) + "\n" +
|
||||
"dir=" + shellQuote(buildDir) + "\n" +
|
||||
"if [ ! -d \"$dir/.git\" ]; then\n" +
|
||||
" attempt=0\n" +
|
||||
" while [ \"$attempt\" -lt 3 ]; do\n" +
|
||||
" if git clone \"$repo\" \"$dir\"; then\n" +
|
||||
" exit 0\n" +
|
||||
" fi\n" +
|
||||
" attempt=$((attempt + 1))\n" +
|
||||
" sleep 2\n" +
|
||||
" done\n" +
|
||||
" exit 1\n" +
|
||||
"fi\n" +
|
||||
"origin=\"$(git -C \"$dir\" config --get remote.origin.url || true)\"\n" +
|
||||
"if [ \"$origin\" != \"$repo\" ]; then\n" +
|
||||
" printf 'existing checkout origin does not match configured repository\\nconfigured: %s\\nfound: %s\\n' \"$repo\" \"$origin\" >&2\n" +
|
||||
" exit 1\n" +
|
||||
"fi\n" +
|
||||
"attempt=0\n" +
|
||||
"while [ \"$attempt\" -lt 3 ]; do\n" +
|
||||
" if git -C \"$dir\" fetch --prune origin; then\n" +
|
||||
" exit 0\n" +
|
||||
" fi\n" +
|
||||
" attempt=$((attempt + 1))\n" +
|
||||
" sleep 2\n" +
|
||||
"done\n" +
|
||||
"exit 1\n"
|
||||
}
|
||||
|
||||
// resolveBranchScript prints the remote's default branch short name
|
||||
// (with an "origin/" prefix) via origin/HEAD. It fails closed so a
|
||||
// set-head failure aborts rather than resolving a stale default.
|
||||
func resolveBranchScript(buildDir string) string {
|
||||
return "set -eu\n" +
|
||||
"git -C " + shellQuote(buildDir) + " remote set-head origin --auto >/dev/null\n" +
|
||||
"git -C " + shellQuote(buildDir) + " symbolic-ref --short refs/remotes/origin/HEAD\n"
|
||||
}
|
||||
|
||||
// refExistsScript verifies a remote-tracking ref exists (exit 0) without
|
||||
// emitting output.
|
||||
func refExistsScript(buildDir, ref string) string {
|
||||
return "git -C " + shellQuote(buildDir) + " show-ref --verify --quiet " + shellQuote(ref)
|
||||
}
|
||||
|
||||
// checkoutScript moves the local branch to the resolved remote branch
|
||||
// and prints the resolved commit. It fails closed (`set -eu`): before the
|
||||
// destructive `checkout -B` (which would silently discard local changes)
|
||||
// it refuses when the tracked working tree is dirty, so a checkout
|
||||
// failure can never be masked by a stale `rev-parse` from the previous
|
||||
// checkout, and a failed checkout aborts before the build.
|
||||
func checkoutScript(buildDir, branch string) string {
|
||||
return "set -eu\n" +
|
||||
"git -C " + shellQuote(buildDir) + " diff --quiet || { echo 'working tree has uncommitted changes; refusing to overwrite' >&2; exit 1; }\n" +
|
||||
"git -C " + shellQuote(buildDir) + " diff --cached --quiet || { echo 'working tree has staged changes; refusing to overwrite' >&2; exit 1; }\n" +
|
||||
"git -C " + shellQuote(buildDir) + " checkout -q -B " + shellQuote(branch) + " " + shellQuote("origin/"+branch) + "\n" +
|
||||
"git -C " + shellQuote(buildDir) + " rev-parse HEAD\n"
|
||||
}
|
||||
|
||||
// commitRecordScript writes the resolved branch and commit to the build
|
||||
// dir record file with a bounded, greppable format. It runs only after a
|
||||
// successful build, so the record always matches the staged binary.
|
||||
func commitRecordScript(buildDir, branch, commit string) string {
|
||||
path := shellQuote(filepath.Join(buildDir, commitRecordName))
|
||||
format := shellQuote("branch=%s\\ncommit=%s\\n")
|
||||
return "umask 022; printf " + format + " " + shellQuote(branch) + " " + shellQuote(commit) +
|
||||
" > " + path + " && chmod 0644 " + path
|
||||
}
|
||||
|
||||
// buildScript builds the worker with the repository's own flags into the
|
||||
// staging path and verifies the resulting binary runs. CGO is disabled,
|
||||
// trimpath keeps the build reproducible, and both caches (GOCACHE and
|
||||
// GOMODCACHE) live inside the build dir so reruns reuse them. The binary
|
||||
// is built to a sibling temp path, verified, and only then atomically
|
||||
// swapped over the previous staging binary, so a failed build never
|
||||
// replaces it.
|
||||
func buildScript(toolchainDir, buildDir, goproxy, stage, ldflags string) string {
|
||||
var b strings.Builder
|
||||
b.WriteString("set -eu\n")
|
||||
b.WriteString("cd " + shellQuote(buildDir) + "\n")
|
||||
b.WriteString("export PATH=" + shellQuote(toolchainDir+"/bin") + ":$PATH\n")
|
||||
b.WriteString("export GOCACHE=" + shellQuote(buildDir+"/.gocache") + "\n")
|
||||
b.WriteString("export GOMODCACHE=" + shellQuote(buildDir+"/.gomodcache") + "\n")
|
||||
if goproxy != "" {
|
||||
b.WriteString("export GOPROXY=" + shellQuote(goproxy) + "\n")
|
||||
}
|
||||
b.WriteString("stage=" + shellQuote(stage) + "\n")
|
||||
b.WriteString("tmp=\"$stage.new\"\n")
|
||||
b.WriteString("trap 'rm -f \"$tmp\"' EXIT HUP INT TERM\n")
|
||||
b.WriteString("CGO_ENABLED=0 ")
|
||||
b.WriteString(shellQuote(toolchainDir + "/bin/go"))
|
||||
b.WriteString(" build -trimpath -ldflags=" + shellQuote(ldflags) + " -o \"$tmp\" ./cmd/rsmon-worker\n")
|
||||
b.WriteString("\"$tmp\" --version\n")
|
||||
b.WriteString("mv -f \"$tmp\" \"$stage\"\n")
|
||||
return b.String()
|
||||
}
|
||||
536
internal/installer/sourceinstall_ssh_test.go
Обычный файл
536
internal/installer/sourceinstall_ssh_test.go
Обычный файл
@@ -0,0 +1,536 @@
|
||||
package installer
|
||||
|
||||
import (
|
||||
"crypto/ed25519"
|
||||
"crypto/rand"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"golang.org/x/crypto/ssh"
|
||||
|
||||
"rocketgit.ru/rsmon/worker/internal/sshinstall"
|
||||
)
|
||||
|
||||
const (
|
||||
fakeSSHPassword = "fake-ssh-password"
|
||||
fakeCommitHex = "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef"
|
||||
fakeOsRelease = "ID=ubuntu\nNAME=\"Ubuntu\"\nVERSION_ID=24.04\n"
|
||||
)
|
||||
|
||||
// fakeSSHServer is a minimal in-process SSH server that simulates a
|
||||
// remote Linux host for the SourceInstall orchestration tests. It uses
|
||||
// real golang.org/x/crypto/ssh transport (no mocked SSH library), so the
|
||||
// executor's dial, session, exec, and stdout/stderr plumbing is
|
||||
// exercised end to end, and it records every command it ran.
|
||||
type fakeSSHServer struct {
|
||||
addr string
|
||||
onExec func(command string) (stdout, stderr string, code int)
|
||||
mu sync.Mutex
|
||||
commands []string
|
||||
}
|
||||
|
||||
func startFakeSSHServer(t *testing.T, onExec func(command string) (string, string, int)) *fakeSSHServer {
|
||||
t.Helper()
|
||||
_, priv, err := ed25519.GenerateKey(rand.Reader)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
signer, err := ssh.NewSignerFromKey(priv)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
config := &ssh.ServerConfig{
|
||||
PasswordCallback: func(_ ssh.ConnMetadata, pass []byte) (*ssh.Permissions, error) {
|
||||
if string(pass) == fakeSSHPassword {
|
||||
return nil, nil
|
||||
}
|
||||
return nil, fmt.Errorf("password rejected")
|
||||
},
|
||||
}
|
||||
config.AddHostKey(signer)
|
||||
|
||||
ln, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = ln.Close() })
|
||||
|
||||
srv := &fakeSSHServer{addr: ln.Addr().String(), onExec: onExec}
|
||||
go func() {
|
||||
for {
|
||||
conn, err := ln.Accept()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
go srv.handleConn(conn, config)
|
||||
}
|
||||
}()
|
||||
return srv
|
||||
}
|
||||
|
||||
func (s *fakeSSHServer) Port() int {
|
||||
_, port, err := net.SplitHostPort(s.addr)
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
p := 0
|
||||
fmt.Sscanf(port, "%d", &p)
|
||||
return p
|
||||
}
|
||||
|
||||
func (s *fakeSSHServer) Commands() []string {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return append([]string(nil), s.commands...)
|
||||
}
|
||||
|
||||
func (s *fakeSSHServer) handleConn(conn net.Conn, config *ssh.ServerConfig) {
|
||||
sconn, chans, reqs, err := ssh.NewServerConn(conn, config)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer sconn.Close() //nolint:errcheck
|
||||
go ssh.DiscardRequests(reqs)
|
||||
for newChannel := range chans {
|
||||
if newChannel.ChannelType() != "session" {
|
||||
_ = newChannel.Reject(ssh.UnknownChannelType, "unknown channel type")
|
||||
continue
|
||||
}
|
||||
channel, requests, err := newChannel.Accept()
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
go func() {
|
||||
defer channel.Close()
|
||||
// Drain the client's stdin so uploads (which stream base64
|
||||
// over the session stdin) never block the channel window.
|
||||
// The channel is not closed until the client's write side is
|
||||
// exhausted, mirroring the real server behavior.
|
||||
var drained sync.WaitGroup
|
||||
drained.Add(1)
|
||||
go func() {
|
||||
defer drained.Done()
|
||||
_, _ = io.Copy(io.Discard, channel)
|
||||
}()
|
||||
s.handleSession(channel, requests)
|
||||
drained.Wait()
|
||||
}()
|
||||
}
|
||||
}
|
||||
|
||||
func (s *fakeSSHServer) handleSession(channel ssh.Channel, requests <-chan *ssh.Request) {
|
||||
for req := range requests {
|
||||
if req.Type != "exec" {
|
||||
if req.WantReply {
|
||||
_ = req.Reply(false, nil)
|
||||
}
|
||||
continue
|
||||
}
|
||||
var payload struct{ Command string }
|
||||
if err := ssh.Unmarshal(req.Payload, &payload); err != nil {
|
||||
if req.WantReply {
|
||||
_ = req.Reply(false, nil)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if req.WantReply {
|
||||
_ = req.Reply(true, nil)
|
||||
}
|
||||
s.mu.Lock()
|
||||
s.commands = append(s.commands, payload.Command)
|
||||
s.mu.Unlock()
|
||||
s.execCommand(channel, payload.Command)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
func (s *fakeSSHServer) execCommand(channel ssh.Channel, command string) {
|
||||
handler := s.onExec
|
||||
if handler == nil {
|
||||
handler = defaultFakeExec
|
||||
}
|
||||
stdout, stderr, code := handler(command)
|
||||
_, _ = channel.Write([]byte(stdout))
|
||||
_, _ = channel.Stderr().Write([]byte(stderr))
|
||||
_, _ = channel.SendRequest("exit-status", false, ssh.Marshal(struct{ Status uint32 }{uint32(code)}))
|
||||
_ = channel.CloseWrite()
|
||||
}
|
||||
|
||||
// defaultFakeExec simulates a minimal Linux host: it answers os-release,
|
||||
// uname, the init-marker file probe, origin URL, branch resolution, and
|
||||
// rev-parse, accepts every install step, and simulates the work-package-4
|
||||
// activation (running-init probe, env/unit uploads, and the activate
|
||||
// script succeeding). A pinned-branch existence check (show-ref) fails by
|
||||
// default so the missing-branch path is exercised without extra setup.
|
||||
func defaultFakeExec(command string) (string, string, int) {
|
||||
switch {
|
||||
case strings.Contains(command, "cat /etc/os-release"):
|
||||
return fakeOsRelease, "", 0
|
||||
case strings.Contains(command, "uname -m"):
|
||||
return "x86_64\n", "", 0
|
||||
case strings.Contains(command, "[ -e"):
|
||||
return "/usr/lib/systemd/system\n", "", 0
|
||||
case strings.Contains(command, "remote.origin.url"):
|
||||
return sshinstall.DefaultRepo + "\n", "", 0
|
||||
case strings.Contains(command, "symbolic-ref"):
|
||||
return "origin/master\n", "", 0
|
||||
case strings.Contains(command, "rev-parse HEAD"):
|
||||
return fakeCommitHex + "\n", "", 0
|
||||
case strings.Contains(command, "show-ref"):
|
||||
return "", "branch not found", 1
|
||||
case strings.Contains(command, "softlevel"):
|
||||
// running-init probe: no systemd/openrc is actually booted.
|
||||
return "none\n", "", 0
|
||||
case strings.Contains(command, "mktemp -d /tmp/rsmon-worker-act"):
|
||||
// server-side secure 0700 upload dir.
|
||||
return "/tmp/rsmon-worker-act-fake\n", "", 0
|
||||
case strings.Contains(command, "base64 -d"):
|
||||
// env/unit uploads succeed.
|
||||
return "", "", 0
|
||||
case strings.Contains(command, "rsmon-worker activated"):
|
||||
// the activate script completed successfully.
|
||||
return "", "", 0
|
||||
default:
|
||||
return "", "", 0
|
||||
}
|
||||
}
|
||||
|
||||
func testSSHOptions(port int) SourceInstallOptions {
|
||||
return SourceInstallOptions{
|
||||
SSHOptions: SSHOptions{
|
||||
Host: "127.0.0.1",
|
||||
Port: port,
|
||||
User: "root",
|
||||
Password: fakeSSHPassword,
|
||||
InsecureHostKey: true,
|
||||
},
|
||||
// Staging-only by default so the work-package-3 orchestration
|
||||
// tests keep their historical shape; activation tests opt in.
|
||||
Activation: ActivationOptions{Activate: false},
|
||||
}
|
||||
}
|
||||
|
||||
// testActivationOptions wraps testSSHOptions with the credentials and
|
||||
// activation flag needed to run the work-package-4 flow against the fake
|
||||
// server.
|
||||
func testActivationOptions(port int) SourceInstallOptions {
|
||||
opts := testSSHOptions(port)
|
||||
opts.Activation = ActivationOptions{
|
||||
Activate: true,
|
||||
URL: "https://rsmon.ru",
|
||||
Token: fakeSSHPassword + "-token",
|
||||
}
|
||||
return opts
|
||||
}
|
||||
|
||||
func TestSourceInstallSSHFlow(t *testing.T) {
|
||||
srv := startFakeSSHServer(t, nil)
|
||||
res, err := SourceInstall(testSSHOptions(srv.Port()))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if res.Detection.Distro != sshinstall.DistroUbuntu || res.Detection.PackageManager != sshinstall.PkgApt {
|
||||
t.Fatalf("detection = %+v", res.Detection)
|
||||
}
|
||||
if res.Detection.InitSystem != sshinstall.InitSystemd {
|
||||
t.Fatalf("init detection = %q, want systemd", res.Detection.InitSystem)
|
||||
}
|
||||
if res.Plan.Toolchain.Arch != "linux-amd64" || res.Plan.Toolchain.Version != "1.26.0" {
|
||||
t.Fatalf("toolchain = %+v", res.Plan.Toolchain)
|
||||
}
|
||||
if len(res.Plan.Packages) == 0 || res.Plan.Repo == "" {
|
||||
t.Fatalf("plan = %+v", res.Plan)
|
||||
}
|
||||
if res.ResolvedBranch != "master" || res.ResolvedCommit != fakeCommitHex {
|
||||
t.Fatalf("resolved = %s @ %s", res.ResolvedBranch, res.ResolvedCommit)
|
||||
}
|
||||
if res.ToolchainDir != "/usr/local/go" || res.StageBinary != "/opt/rsmon-worker-src/rsmon-worker" ||
|
||||
res.RecordFile != "/opt/rsmon-worker-src/rsmon-worker.commit" {
|
||||
t.Fatalf("paths = %+v", res)
|
||||
}
|
||||
|
||||
commands := srv.Commands()
|
||||
var joined strings.Builder
|
||||
for _, c := range commands {
|
||||
joined.WriteString(c)
|
||||
joined.WriteString("\n")
|
||||
}
|
||||
// Marker substrings that survive the nested `sh -c '<script>'`
|
||||
// quoting; exact quoting of each script is asserted by the unit
|
||||
// tests (TestPackageScript, TestToolchainScript, ...).
|
||||
for _, want := range []string{
|
||||
"cat /etc/os-release",
|
||||
"uname -m",
|
||||
"apt-get update",
|
||||
"apt-get install -y --no-install-recommends",
|
||||
"sha256sum -c -",
|
||||
"git clone",
|
||||
"git -C",
|
||||
"fetch --prune origin",
|
||||
"symbolic-ref --short refs/remotes/origin/HEAD",
|
||||
"checkout -q -B",
|
||||
"rev-parse HEAD",
|
||||
"branch=%s\\ncommit=%s\\n",
|
||||
fakeCommitHex,
|
||||
"CGO_ENABLED=0",
|
||||
"build -trimpath",
|
||||
"GOMODCACHE",
|
||||
"mv -f",
|
||||
} {
|
||||
if !strings.Contains(joined.String(), want) {
|
||||
t.Fatalf("recorded commands missing %q:\n%s", want, joined.String())
|
||||
}
|
||||
}
|
||||
|
||||
// Ordering: prerequisites before toolchain before source before build.
|
||||
idx := func(sub string) int {
|
||||
for i, c := range commands {
|
||||
if strings.Contains(c, sub) {
|
||||
return i
|
||||
}
|
||||
}
|
||||
t.Fatalf("command %q not found in %v", sub, commands)
|
||||
return -1
|
||||
}
|
||||
if !(idx("apt-get install") < idx("sha256sum") && idx("sha256sum") < idx("git clone") &&
|
||||
idx("git clone") < idx("rev-parse") && idx("rev-parse") < idx("-trimpath") &&
|
||||
idx("-trimpath") < idx("branch=%s")) {
|
||||
t.Fatalf("step order wrong: %v", commands)
|
||||
}
|
||||
|
||||
// Adaptive resolution means no pinned-branch existence check ran.
|
||||
if strings.Contains(joined.String(), "show-ref") {
|
||||
t.Fatalf("show-ref ran despite branch resolution:\n%s", joined.String())
|
||||
}
|
||||
if strings.Contains(joined.String(), fakeSSHPassword) {
|
||||
t.Fatal("SSH password leaked into a remote command")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSourceInstallSSHRejectsMissingPinnedBranch(t *testing.T) {
|
||||
srv := startFakeSSHServer(t, nil)
|
||||
opts := testSSHOptions(srv.Port())
|
||||
opts.Branch = "main"
|
||||
_, err := SourceInstall(opts)
|
||||
if err == nil || !strings.Contains(err.Error(), `branch "main" does not exist`) {
|
||||
t.Fatalf("err = %v, want missing-branch error", err)
|
||||
}
|
||||
if commands := srv.Commands(); !strings.Contains(strings.Join(commands, "\n"), "show-ref") {
|
||||
t.Fatalf("pinned branch existence was not verified: %v", commands)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSourceInstallSSHBuildFailure(t *testing.T) {
|
||||
srv := startFakeSSHServer(t, func(command string) (string, string, int) {
|
||||
if strings.Contains(command, "-trimpath") {
|
||||
return "", "build exploded", 1
|
||||
}
|
||||
return defaultFakeExec(command)
|
||||
})
|
||||
_, err := SourceInstall(testSSHOptions(srv.Port()))
|
||||
if err == nil || !strings.Contains(err.Error(), "build worker binary") || !strings.Contains(err.Error(), "build exploded") {
|
||||
t.Fatalf("err = %v, want bounded build failure", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSourceInstallSSHDetectionFailure(t *testing.T) {
|
||||
srv := startFakeSSHServer(t, func(command string) (string, string, int) {
|
||||
if strings.Contains(command, "os-release") {
|
||||
return "", "os-release unreadable", 1
|
||||
}
|
||||
return defaultFakeExec(command)
|
||||
})
|
||||
_, err := SourceInstall(testSSHOptions(srv.Port()))
|
||||
if err == nil || !strings.Contains(err.Error(), "/etc/os-release") {
|
||||
t.Fatalf("err = %v, want detection failure", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestSourceInstallSSHCheckoutFailureNotMasked proves the fail-closed
|
||||
// contract: a failed checkout (e.g. a dirty working tree) surfaces as an
|
||||
// error and never reaches the build or commit-record steps, so the
|
||||
// previous staging binary and record are preserved.
|
||||
func TestSourceInstallSSHCheckoutFailureNotMasked(t *testing.T) {
|
||||
srv := startFakeSSHServer(t, func(command string) (string, string, int) {
|
||||
if strings.Contains(command, "checkout -q -B") {
|
||||
return "", "your local changes to the following files would be overwritten by checkout", 1
|
||||
}
|
||||
return defaultFakeExec(command)
|
||||
})
|
||||
_, err := SourceInstall(testSSHOptions(srv.Port()))
|
||||
if err == nil || !strings.Contains(err.Error(), "check out branch") {
|
||||
t.Fatalf("err = %v, want checkout failure", err)
|
||||
}
|
||||
joined := strings.Join(srv.Commands(), "\n")
|
||||
if strings.Contains(joined, "-trimpath") || strings.Contains(joined, "branch=%s") {
|
||||
t.Fatalf("build or record ran after checkout failed:\n%s", joined)
|
||||
}
|
||||
}
|
||||
|
||||
// TestSourceInstallSSHCommandTimeout verifies each remote command is
|
||||
// bounded by SessionTimeout and the run reports it.
|
||||
func TestSourceInstallSSHCommandTimeout(t *testing.T) {
|
||||
srv := startFakeSSHServer(t, func(command string) (string, string, int) {
|
||||
if strings.Contains(command, "uname -m") {
|
||||
time.Sleep(5 * time.Second)
|
||||
return "x86_64\n", "", 0
|
||||
}
|
||||
return defaultFakeExec(command)
|
||||
})
|
||||
opts := testSSHOptions(srv.Port())
|
||||
opts.SessionTimeout = 300 * time.Millisecond
|
||||
_, err := SourceInstall(opts)
|
||||
if err == nil || !strings.Contains(err.Error(), "timed out after") {
|
||||
t.Fatalf("err = %v, want command timeout", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestSourceInstallSSHActivationFlow verifies the work-package-4 flow
|
||||
// over a real SSH session: after the staging build and commit record, the
|
||||
// installer probes the running init, uploads the env and service
|
||||
// definition, runs the atomic activation script, and reports the installed
|
||||
// layout. The secrets-absent contract holds: the worker token never
|
||||
// reaches a remote command.
|
||||
func TestSourceInstallSSHActivationFlow(t *testing.T) {
|
||||
srv := startFakeSSHServer(t, nil)
|
||||
res, err := SourceInstall(testActivationOptions(srv.Port()))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if res.Activation == nil {
|
||||
t.Fatal("activation result missing")
|
||||
}
|
||||
if res.Activation.Binary != "/usr/local/bin/rsmon-worker" ||
|
||||
res.Activation.EnvFile != "/etc/rsmon-worker/worker.env" ||
|
||||
res.Activation.DataDir != "/var/lib/rsmon-worker" ||
|
||||
res.Activation.UnitFile != "/etc/systemd/system/rsmon-worker.service" ||
|
||||
res.Activation.Supervisor != "none" ||
|
||||
!res.Activation.Started {
|
||||
t.Fatalf("activation result = %+v", res.Activation)
|
||||
}
|
||||
|
||||
commands := srv.Commands()
|
||||
var joined strings.Builder
|
||||
for _, c := range commands {
|
||||
joined.WriteString(c)
|
||||
joined.WriteString("\n")
|
||||
}
|
||||
for _, want := range []string{
|
||||
"/run/systemd/system",
|
||||
"softlevel",
|
||||
"mktemp -d /tmp/rsmon-worker-act",
|
||||
"umask 077; base64 -d >",
|
||||
"rsmon-worker activated",
|
||||
"install -m 0755",
|
||||
"install -m 0600",
|
||||
"chmod 0750",
|
||||
"worker.pid",
|
||||
"liveness",
|
||||
"rm -rf --",
|
||||
} {
|
||||
if !strings.Contains(joined.String(), want) {
|
||||
t.Fatalf("activation commands missing %q:\n%s", want, joined.String())
|
||||
}
|
||||
}
|
||||
|
||||
// Ordering: build and commit record, then a server-side secure upload
|
||||
// dir is created, the env/unit are uploaded into it, and the activation
|
||||
// script runs last.
|
||||
idx := func(sub string) int {
|
||||
for i, c := range commands {
|
||||
if strings.Contains(c, sub) {
|
||||
return i
|
||||
}
|
||||
}
|
||||
t.Fatalf("command %q not found in %v", sub, commands)
|
||||
return -1
|
||||
}
|
||||
if idx("branch=%s") >= idx("mktemp -d /tmp/rsmon-worker-act") ||
|
||||
idx("mktemp -d /tmp/rsmon-worker-act") >= idx("base64 -d >") ||
|
||||
idx("base64 -d >") >= idx("rsmon-worker activated") {
|
||||
t.Fatalf("activation order wrong: %v", commands)
|
||||
}
|
||||
if strings.Contains(joined.String(), fakeSSHPassword+"-token") {
|
||||
t.Fatal("worker token leaked into a remote command")
|
||||
}
|
||||
}
|
||||
|
||||
// TestSourceInstallSSHNoActivationSkipsService proves the staging
|
||||
// boundary: without Activation the installer never probes the init system,
|
||||
// uploads an env/unit, or runs the activation script.
|
||||
func TestSourceInstallSSHNoActivationSkipsService(t *testing.T) {
|
||||
srv := startFakeSSHServer(t, nil)
|
||||
res, err := SourceInstall(testSSHOptions(srv.Port()))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if res.Activation != nil {
|
||||
t.Fatalf("activation must not run without the flag: %+v", res.Activation)
|
||||
}
|
||||
joined := strings.Join(srv.Commands(), "\n")
|
||||
for _, forbidden := range []string{"softlevel", "base64 -d >", "rsmon-worker activated", "worker.pid"} {
|
||||
if strings.Contains(joined, forbidden) {
|
||||
t.Fatalf("staging-only flow ran activation step %q:\n%s", forbidden, joined)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestSourceInstallSSHActivationRequiresToken verifies the credentials
|
||||
// gate: activation without a token fails before any remote connection.
|
||||
func TestSourceInstallSSHActivationRequiresToken(t *testing.T) {
|
||||
srv := startFakeSSHServer(t, nil)
|
||||
opts := testSSHOptions(srv.Port())
|
||||
opts.Activation = ActivationOptions{Activate: true, URL: "https://rsmon.ru"}
|
||||
_, err := SourceInstall(opts)
|
||||
if err == nil || !strings.Contains(err.Error(), "RSMON_TOKEN") {
|
||||
t.Fatalf("err = %v, want token requirement", err)
|
||||
}
|
||||
if got := len(srv.Commands()); got != 0 {
|
||||
t.Fatalf("commands ran before validation failed: %d", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestSourceInstallSSHActivationFailure verifies the bounded failure
|
||||
// path: a failing activate script surfaces as a labelled error with the
|
||||
// remote stderr, and the uploaded temp env/unit are cleaned up by the
|
||||
// controller defer regardless of the outcome.
|
||||
func TestSourceInstallSSHActivationFailure(t *testing.T) {
|
||||
srv := startFakeSSHServer(t, func(command string) (string, string, int) {
|
||||
if strings.Contains(command, "rsmon-worker activated") {
|
||||
return "", "activation exploded: disk full", 1
|
||||
}
|
||||
return defaultFakeExec(command)
|
||||
})
|
||||
_, err := SourceInstall(testActivationOptions(srv.Port()))
|
||||
if err == nil || !strings.Contains(err.Error(), "activate worker service") || !strings.Contains(err.Error(), "disk full") {
|
||||
t.Fatalf("err = %v, want bounded activation failure", err)
|
||||
}
|
||||
joined := strings.Join(srv.Commands(), "\n")
|
||||
if !strings.Contains(joined, "rm -rf --") {
|
||||
t.Fatalf("secure upload dir cleanup not issued after activation failure:\n%s", joined)
|
||||
}
|
||||
}
|
||||
|
||||
// TestSourceInstallSSHActivationHealthFailure verifies the health gate
|
||||
// surfaces as an activation failure and the temp files are still cleaned.
|
||||
// The fake cannot run the shell script, so the failure is injected at the
|
||||
// activate-command level; the real rollback semantics are covered by the
|
||||
// Docker/OpenSSH E2E tests.
|
||||
func TestSourceInstallSSHActivationHealthFailure(t *testing.T) {
|
||||
srv := startFakeSSHServer(t, func(command string) (string, string, int) {
|
||||
if strings.Contains(command, "rsmon-worker activated") {
|
||||
return "", "health failure: worker did not answer /healthz", 1
|
||||
}
|
||||
return defaultFakeExec(command)
|
||||
})
|
||||
_, err := SourceInstall(testActivationOptions(srv.Port()))
|
||||
if err == nil || !strings.Contains(err.Error(), "did not answer /healthz") {
|
||||
t.Fatalf("err = %v, want /healthz verification failure", err)
|
||||
}
|
||||
}
|
||||
395
internal/installer/sourceinstall_test.go
Обычный файл
395
internal/installer/sourceinstall_test.go
Обычный файл
@@ -0,0 +1,395 @@
|
||||
package installer
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"rocketgit.ru/rsmon/worker/internal/sshinstall"
|
||||
)
|
||||
|
||||
// cannedSHA is a fixed 64-hex value used to exercise script rendering.
|
||||
const cannedSHA = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
|
||||
|
||||
func TestPackageScript(t *testing.T) {
|
||||
pkgs := []string{"git", "ca-certificates", "curl", "tar", "gzip"}
|
||||
|
||||
apk := packageScript(sshinstall.PkgApk, pkgs)
|
||||
if !strings.HasPrefix(apk, "apk add --no-cache ") {
|
||||
t.Fatalf("apk script = %q", apk)
|
||||
}
|
||||
for _, p := range pkgs {
|
||||
if !strings.Contains(apk, shellQuote(p)) {
|
||||
t.Fatalf("apk script missing quoted package %q: %q", p, apk)
|
||||
}
|
||||
}
|
||||
|
||||
apt := packageScript(sshinstall.PkgApt, pkgs)
|
||||
for _, want := range []string{"export DEBIAN_FRONTEND=noninteractive", "apt-get update", "apt-get install -y --no-install-recommends"} {
|
||||
if !strings.Contains(apt, want) {
|
||||
t.Fatalf("apt script missing %q: %q", want, apt)
|
||||
}
|
||||
}
|
||||
|
||||
pacman := packageScript(sshinstall.PkgPacman, pkgs)
|
||||
if !strings.Contains(pacman, "pacman -Sy --noconfirm --needed") {
|
||||
t.Fatalf("pacman script = %q", pacman)
|
||||
}
|
||||
|
||||
dnf := packageScript(sshinstall.PkgDnf, pkgs)
|
||||
if !strings.Contains(dnf, "dnf install -y") {
|
||||
t.Fatalf("dnf script = %q", dnf)
|
||||
}
|
||||
|
||||
if got := packageScript(sshinstall.PkgUnknown, pkgs); got != "" {
|
||||
t.Fatalf("unknown pkg script = %q, want empty", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPackageScriptNeverIncludesCompiler(t *testing.T) {
|
||||
pkgs := []string{"git", "ca-certificates", "curl", "tar", "gzip"}
|
||||
for _, pkg := range []sshinstall.PackageManager{sshinstall.PkgApk, sshinstall.PkgApt, sshinstall.PkgPacman, sshinstall.PkgDnf} {
|
||||
script := packageScript(pkg, pkgs)
|
||||
for _, bad := range []string{"build-essential", "gcc", "g++", "base-devel", "make", "gcc-c++"} {
|
||||
if strings.Contains(script, bad) {
|
||||
t.Fatalf("script for %s includes compiler hint %q: %q", pkg, bad, script)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestToolchainScript(t *testing.T) {
|
||||
tc := sshinstall.Toolchain{
|
||||
Version: "1.26.0",
|
||||
Arch: "linux-amd64",
|
||||
URL: "https://go.dev/dl/go1.26.0.linux-amd64.tar.gz",
|
||||
SHA256: cannedSHA,
|
||||
}
|
||||
script := toolchainScript(tc, "/usr/local/go")
|
||||
for _, want := range []string{
|
||||
"set -eu",
|
||||
"mktemp -d /tmp/rsmon-toolchain-XXXXXX",
|
||||
"mktemp -d \"$parent/.go-staging-XXXXXX\"",
|
||||
"trap 'rm -rf \"$work\" \"$staging\"' EXIT HUP INT TERM",
|
||||
"curl -fsSL --retry 3 --retry-delay 2 -o \"$archive\" 'https://go.dev/dl/go1.26.0.linux-amd64.tar.gz'",
|
||||
"sha256sum -c -",
|
||||
cannedSHA,
|
||||
"tar -C \"$staging\" -xzf \"$archive\"",
|
||||
"staged=\"$($staging/go/bin/go version | awk '{print $3}')\"",
|
||||
"backup=\"$parent/.go-backup\"",
|
||||
"mv \"$toolchain\" \"$backup\"",
|
||||
"mv \"$staging/go\" \"$toolchain\"",
|
||||
"\"$toolchain/bin/go\" version",
|
||||
"go toolchain already present",
|
||||
} {
|
||||
if !strings.Contains(script, want) {
|
||||
t.Fatalf("toolchain script missing %q:\n%s", want, script)
|
||||
}
|
||||
}
|
||||
if !strings.Contains(script, shellQuote("go1.26.0")) {
|
||||
t.Fatalf("toolchain script missing version guard:\n%s", script)
|
||||
}
|
||||
}
|
||||
|
||||
func TestToolchainScriptIdempotentSkipOnlyForMatchingVersion(t *testing.T) {
|
||||
script := toolchainScript(sshinstall.Toolchain{Version: "1.26.0", Arch: "linux-amd64", URL: "u", SHA256: cannedSHA}, "/usr/local/go")
|
||||
if strings.Count(script, "exit 0") != 1 {
|
||||
t.Fatalf("toolchain script should skip only once:\n%s", script)
|
||||
}
|
||||
// Replacement must be atomic: the prior toolchain is moved aside and
|
||||
// restored when the swap fails, never removed before staging is ready.
|
||||
for _, want := range []string{
|
||||
"if [ -e \"$toolchain\" ]; then",
|
||||
"mv \"$toolchain\" \"$backup\"",
|
||||
"if ! mv \"$staging/go\" \"$toolchain\"; then",
|
||||
"mv \"$backup\" \"$toolchain\"",
|
||||
} {
|
||||
if !strings.Contains(script, want) {
|
||||
t.Fatalf("toolchain script missing %q:\n%s", want, script)
|
||||
}
|
||||
}
|
||||
if strings.Contains(script, "rm -rf \"$toolchain\"") {
|
||||
t.Fatalf("toolchain script must not delete the live toolchain directly:\n%s", script)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCloneUpdateScript(t *testing.T) {
|
||||
script := cloneUpdateScript("https://example.test/worker.git", "/opt/rsmon-worker-src")
|
||||
for _, want := range []string{
|
||||
"set -u",
|
||||
"[ ! -d \"$dir/.git\" ]",
|
||||
"git clone \"$repo\" \"$dir\"",
|
||||
"while [ \"$attempt\" -lt 3 ]",
|
||||
"sleep 2",
|
||||
"git -C \"$dir\" config --get remote.origin.url",
|
||||
"existing checkout origin does not match configured repository",
|
||||
"git -C \"$dir\" fetch --prune origin",
|
||||
"exit 1",
|
||||
} {
|
||||
if !strings.Contains(script, want) {
|
||||
t.Fatalf("clone/update script missing %q:\n%s", want, script)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCheckoutScriptFailsClosed(t *testing.T) {
|
||||
script := checkoutScript("/opt/rsmon-worker-src", "master")
|
||||
if !strings.HasPrefix(script, "set -eu\n") {
|
||||
t.Fatalf("checkout script must fail closed with set -eu:\n%s", script)
|
||||
}
|
||||
for _, want := range []string{
|
||||
"git -C '/opt/rsmon-worker-src' diff --quiet ||",
|
||||
"git -C '/opt/rsmon-worker-src' diff --cached --quiet ||",
|
||||
"refusing to overwrite",
|
||||
"checkout -q -B 'master' 'origin/master'",
|
||||
"rev-parse HEAD",
|
||||
} {
|
||||
if !strings.Contains(script, want) {
|
||||
t.Fatalf("checkout script missing %q:\n%s", want, script)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveBranchScriptFailsClosed(t *testing.T) {
|
||||
if !strings.HasPrefix(resolveBranchScript("/opt/rsmon-worker-src"), "set -eu\n") {
|
||||
t.Fatalf("resolve-branch script must fail closed:\n%s", resolveBranchScript("/opt/rsmon-worker-src"))
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveBranchScript(t *testing.T) {
|
||||
script := resolveBranchScript("/opt/rsmon-worker-src")
|
||||
for _, want := range []string{
|
||||
"remote set-head origin --auto",
|
||||
"symbolic-ref --short refs/remotes/origin/HEAD",
|
||||
} {
|
||||
if !strings.Contains(script, want) {
|
||||
t.Fatalf("resolve-branch script missing %q:\n%s", want, script)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCheckoutScript(t *testing.T) {
|
||||
script := checkoutScript("/opt/rsmon-worker-src", "master")
|
||||
for _, want := range []string{
|
||||
"checkout -q -B 'master' 'origin/master'",
|
||||
"rev-parse HEAD",
|
||||
} {
|
||||
if !strings.Contains(script, want) {
|
||||
t.Fatalf("checkout script missing %q:\n%s", want, script)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRefExistsScript(t *testing.T) {
|
||||
script := refExistsScript("/opt/rsmon-worker-src", "refs/remotes/origin/main")
|
||||
if !strings.Contains(script, "show-ref --verify --quiet 'refs/remotes/origin/main'") {
|
||||
t.Fatalf("ref-exists script = %q", script)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCommitRecordScript(t *testing.T) {
|
||||
script := commitRecordScript("/opt/rsmon-worker-src", "master", strings.Repeat("a", 40))
|
||||
for _, want := range []string{
|
||||
"branch=%s\\ncommit=%s\\n",
|
||||
"'master'",
|
||||
strings.Repeat("a", 40),
|
||||
"'/opt/rsmon-worker-src/rsmon-worker.commit'",
|
||||
"chmod 0644",
|
||||
} {
|
||||
if !strings.Contains(script, want) {
|
||||
t.Fatalf("record script missing %q:\n%s", want, script)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildScript(t *testing.T) {
|
||||
script := buildScript("/usr/local/go", "/opt/rsmon-worker-src", "", "/opt/rsmon-worker-src/rsmon-worker",
|
||||
`-s -w -X main.version=dev -X main.commit=abcdef012345 -X main.buildDate=2026-08-12T00:00:00Z`)
|
||||
for _, want := range []string{
|
||||
"set -eu",
|
||||
"cd '/opt/rsmon-worker-src'",
|
||||
"export PATH='/usr/local/go/bin':$PATH",
|
||||
"export GOCACHE='/opt/rsmon-worker-src/.gocache'",
|
||||
"export GOMODCACHE='/opt/rsmon-worker-src/.gomodcache'",
|
||||
"CGO_ENABLED=0 '/usr/local/go/bin/go' build -trimpath",
|
||||
"-X main.commit=abcdef012345",
|
||||
"-o \"$tmp\" ./cmd/rsmon-worker",
|
||||
"tmp=\"$stage.new\"",
|
||||
"\"$tmp\" --version",
|
||||
"mv -f \"$tmp\" \"$stage\"",
|
||||
"trap 'rm -f \"$tmp\"' EXIT HUP INT TERM",
|
||||
} {
|
||||
if !strings.Contains(script, want) {
|
||||
t.Fatalf("build script missing %q:\n%s", want, script)
|
||||
}
|
||||
}
|
||||
if strings.Contains(script, "GOPROXY") {
|
||||
t.Fatalf("empty GOPROXY must not be exported:\n%s", script)
|
||||
}
|
||||
|
||||
withProxy := buildScript("/usr/local/go", "/opt/rsmon-worker-src", "https://proxy.golang.org,direct", "/opt/rsmon-worker-src/rsmon-worker", "-s -w")
|
||||
if !strings.Contains(withProxy, "export GOPROXY='https://proxy.golang.org,direct'") {
|
||||
t.Fatalf("GOPROXY override not rendered:\n%s", withProxy)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildScriptVerifiesBeforeSwap(t *testing.T) {
|
||||
script := buildScript("/usr/local/go", "/opt/rsmon-worker-src", "", "/opt/rsmon-worker-src/rsmon-worker", "-s -w")
|
||||
verify := strings.Index(script, "\"$tmp\" --version")
|
||||
swap := strings.Index(script, "mv -f \"$tmp\" \"$stage\"")
|
||||
if verify < 0 || swap < 0 || verify > swap {
|
||||
t.Fatalf("build script must verify the temp binary before swapping it in:\n%s", script)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseResolvedCommit(t *testing.T) {
|
||||
commit := strings.Repeat("abcdef", 6) + "abcd" // 40 hex
|
||||
if got, err := parseResolvedCommit(commit + "\n"); err != nil || got != commit {
|
||||
t.Fatalf("parseResolvedCommit() = %q, %v", got, err)
|
||||
}
|
||||
if got, err := parseResolvedCommit("ignored\n" + commit + "\n"); err != nil || got != commit {
|
||||
t.Fatalf("parseResolvedCommit() multi-line = %q, %v", got, err)
|
||||
}
|
||||
for _, bad := range []string{"", "abc", strings.Repeat("A", 40), strings.Repeat("a", 39), "x" + strings.Repeat("a", 39)} {
|
||||
if _, err := parseResolvedCommit(bad); err == nil {
|
||||
t.Fatalf("parseResolvedCommit(%q) succeeded", bad)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseResolvedBranch(t *testing.T) {
|
||||
if got, err := parseResolvedBranch("origin/master\n"); err != nil || got != "master" {
|
||||
t.Fatalf("parseResolvedBranch(origin/master) = %q, %v", got, err)
|
||||
}
|
||||
if got, err := parseResolvedBranch("master\n"); err != nil || got != "master" {
|
||||
t.Fatalf("parseResolvedBranch(master) = %q, %v", got, err)
|
||||
}
|
||||
// A multi-component short name is legal git and stays safe because
|
||||
// the value is validated and single-quoted everywhere it is used.
|
||||
if got, err := parseResolvedBranch("origin/release/1.0\n"); err != nil || got != "release/1.0" {
|
||||
t.Fatalf("parseResolvedBranch(nested) = %q, %v", got, err)
|
||||
}
|
||||
for _, bad := range []string{"origin/../evil\n", "origin/x y\n", "origin/x..y\n", "origin/x@{y\n", "origin/-x\n"} {
|
||||
if _, err := parseResolvedBranch(bad); err == nil {
|
||||
t.Fatalf("parseResolvedBranch(%q) succeeded", bad)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidBranchName(t *testing.T) {
|
||||
for _, ok := range []string{"main", "master", "release-1.0", "feature/x", "a", "v1.2.3", "a_b", "release/1.0"} {
|
||||
if !validBranchName(ok) {
|
||||
t.Fatalf("validBranchName(%q) rejected", ok)
|
||||
}
|
||||
}
|
||||
for _, bad := range []string{"", "-bad", "x..y", "x@{y", "x y", "/x", "x.", "x/", "x//y", "x\\y", "x;y", "$x", "`x`"} {
|
||||
if validBranchName(bad) {
|
||||
t.Fatalf("validBranchName(%q) accepted", bad)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestQuoteList(t *testing.T) {
|
||||
if got, want := quoteList([]string{"git", "ca-certificates"}), "'git' 'ca-certificates'"; got != want {
|
||||
t.Fatalf("quoteList() = %q, want %q", got, want)
|
||||
}
|
||||
if got := quoteList(nil); got != "" {
|
||||
t.Fatalf("quoteList(nil) = %q, want empty", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeSourceOptions(t *testing.T) {
|
||||
o, err := normalizeSourceOptions(SourceInstallOptions{SSHOptions: SSHOptions{Host: "h", User: "u"}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if o.Port != 22 || o.ToolchainDir != "/usr/local/go" || o.SessionTimeout != defaultSessionTimeout {
|
||||
t.Fatalf("defaults not applied: %+v", o)
|
||||
}
|
||||
if o, err := normalizeSourceOptions(SourceInstallOptions{SSHOptions: SSHOptions{Host: "h", User: "u"}, SessionTimeout: 7 * time.Minute}); err != nil || o.SessionTimeout != 7*time.Minute {
|
||||
t.Fatalf("explicit session timeout not honored: %+v, %v", o, err)
|
||||
}
|
||||
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
opts SourceInstallOptions
|
||||
}{
|
||||
{name: "missing host", opts: SourceInstallOptions{SSHOptions: SSHOptions{User: "u"}}},
|
||||
{name: "missing user", opts: SourceInstallOptions{SSHOptions: SSHOptions{Host: "h"}}},
|
||||
{name: "bad port", opts: SourceInstallOptions{SSHOptions: SSHOptions{Host: "h", User: "u", Port: 70000}}},
|
||||
{name: "bad branch", opts: SourceInstallOptions{SSHOptions: SSHOptions{Host: "h", User: "u"}, Branch: "x y"}},
|
||||
{name: "bad go version", opts: SourceInstallOptions{SSHOptions: SSHOptions{Host: "h", User: "u"}, GoVersion: "1.26;rm"}},
|
||||
{name: "bad go arch", opts: SourceInstallOptions{SSHOptions: SSHOptions{Host: "h", User: "u"}, GoArch: "amd64;rm"}},
|
||||
{name: "bad repo scheme", opts: SourceInstallOptions{SSHOptions: SSHOptions{Host: "h", User: "u"}, Repo: "http://x/y"}},
|
||||
{name: "repo userinfo", opts: SourceInstallOptions{SSHOptions: SSHOptions{Host: "h", User: "u"}, Repo: "https://user:pass@x/y"}},
|
||||
{name: "relative toolchain", opts: SourceInstallOptions{SSHOptions: SSHOptions{Host: "h", User: "u"}, ToolchainDir: "usr/local/go"}},
|
||||
{name: "non-go toolchain basename", opts: SourceInstallOptions{SSHOptions: SSHOptions{Host: "h", User: "u"}, ToolchainDir: "/opt/golang"}},
|
||||
{name: "relative stage", opts: SourceInstallOptions{SSHOptions: SSHOptions{Host: "h", User: "u"}, StageBinary: "bin/worker"}},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
if _, err := normalizeSourceOptions(tc.opts); err == nil {
|
||||
t.Fatalf("normalizeSourceOptions(%s) succeeded", tc.name)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSudoWrap(t *testing.T) {
|
||||
cmd, stdin := sudoWrap("root", "", "echo hi")
|
||||
if cmd != "echo hi" || stdin != nil {
|
||||
t.Fatalf("root wrap = %q, %q", cmd, stdin)
|
||||
}
|
||||
|
||||
cmd, stdin = sudoWrap("deploy", "", "echo hi")
|
||||
if cmd != "sudo -n -- echo hi" || stdin != nil {
|
||||
t.Fatalf("passwordless sudo wrap = %q, %q", cmd, stdin)
|
||||
}
|
||||
|
||||
cmd, stdin = sudoWrap("deploy", "supersecret", "echo hi")
|
||||
if cmd != "sudo -S -p '' -- echo hi" || string(stdin) != "supersecret\n" {
|
||||
t.Fatalf("sudo -S wrap = %q, %q", cmd, stdin)
|
||||
}
|
||||
if strings.Contains(cmd, "supersecret") {
|
||||
t.Fatal("sudo password leaked into the command string")
|
||||
}
|
||||
}
|
||||
|
||||
// TestSourceScriptsNeverContainSecrets documents the "secrets absent"
|
||||
// contract: none of the rendered remote scripts carry a credential.
|
||||
func TestBoundedBuffer(t *testing.T) {
|
||||
var b boundedBuffer
|
||||
b.max = 8
|
||||
if _, err := b.Write([]byte("12345")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if b.truncated {
|
||||
t.Fatal("truncated before exceeding max")
|
||||
}
|
||||
if _, err := b.Write([]byte("6789abcdef")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !b.truncated {
|
||||
t.Fatal("overflow not flagged")
|
||||
}
|
||||
if got, want := b.String(), "12345678"; got != want {
|
||||
t.Fatalf("boundedBuffer = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSourceScriptsNeverContainSecrets(t *testing.T) {
|
||||
const secret = "super-secret-token-value"
|
||||
scripts := []string{
|
||||
packageScript(sshinstall.PkgApk, []string{"git", "ca-certificates", "curl", "tar", "gzip"}),
|
||||
toolchainScript(sshinstall.Toolchain{Version: "1.26.0", Arch: "linux-amd64", URL: "https://go.dev/dl/go1.26.0.linux-amd64.tar.gz", SHA256: cannedSHA}, "/usr/local/go"),
|
||||
cloneUpdateScript("https://example.test/worker.git", "/opt/rsmon-worker-src"),
|
||||
resolveBranchScript("/opt/rsmon-worker-src"),
|
||||
checkoutScript("/opt/rsmon-worker-src", "master"),
|
||||
commitRecordScript("/opt/rsmon-worker-src", "master", strings.Repeat("a", 40)),
|
||||
buildScript("/usr/local/go", "/opt/rsmon-worker-src", "", "/opt/rsmon-worker-src/rsmon-worker", "-s -w"),
|
||||
}
|
||||
for i, script := range scripts {
|
||||
if strings.Contains(script, secret) {
|
||||
t.Fatalf("script %d contains a secret", i)
|
||||
}
|
||||
}
|
||||
}
|
||||
221
internal/sshinstall/detect.go
Обычный файл
221
internal/sshinstall/detect.go
Обычный файл
@@ -0,0 +1,221 @@
|
||||
// Package sshinstall contains the pure detection and planning layer for
|
||||
// installing the worker from source over SSH. It never touches the
|
||||
// network or a remote host; the executor that turns a plan into remote
|
||||
// commands is a later work package (docs/source-installation.md).
|
||||
//
|
||||
// The Docker/OpenSSH harness that exercises detection against real
|
||||
// distro containers lives in internal/installer/harness.
|
||||
package sshinstall
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Distro is a supported remote Linux distribution family.
|
||||
type Distro string
|
||||
|
||||
const (
|
||||
DistroUnknown Distro = "unknown"
|
||||
DistroAlpine Distro = "alpine"
|
||||
DistroDebian Distro = "debian"
|
||||
DistroUbuntu Distro = "ubuntu"
|
||||
DistroArch Distro = "arch"
|
||||
DistroCentOS Distro = "centos"
|
||||
DistroFedora Distro = "fedora"
|
||||
DistroRocky Distro = "rocky"
|
||||
DistroAlma Distro = "alma"
|
||||
DistroRHEL Distro = "rhel"
|
||||
)
|
||||
|
||||
// PackageManager is the remote package manager used to install build
|
||||
// prerequisites.
|
||||
type PackageManager string
|
||||
|
||||
const (
|
||||
PkgUnknown PackageManager = ""
|
||||
PkgApk PackageManager = "apk"
|
||||
PkgApt PackageManager = "apt"
|
||||
PkgPacman PackageManager = "pacman"
|
||||
PkgDnf PackageManager = "dnf"
|
||||
PkgYum PackageManager = "yum"
|
||||
)
|
||||
|
||||
// InitSystem is the remote init system. The source installer must know
|
||||
// it before it can install a service definition; Alpine's OpenRC and the
|
||||
// systemd distros take different paths.
|
||||
type InitSystem string
|
||||
|
||||
const (
|
||||
InitUnknown InitSystem = ""
|
||||
InitSystemd InitSystem = "systemd"
|
||||
InitOpenRC InitSystem = "openrc"
|
||||
)
|
||||
|
||||
// Detection is the resolved identity of a remote Linux host.
|
||||
type Detection struct {
|
||||
ID string // os-release ID (lowercase, e.g. "alpine")
|
||||
Name string // os-release NAME (pretty, may be empty)
|
||||
VersionID string // os-release VERSION_ID (may be empty)
|
||||
Distro Distro
|
||||
PackageManager PackageManager
|
||||
InitSystem InitSystem
|
||||
}
|
||||
|
||||
// FileProber reports which of the given absolute paths exist on the
|
||||
// remote host. Detection uses it to distinguish init systems and
|
||||
// packaging hints without parsing the full command surface. A nil
|
||||
// prober behaves as if nothing exists.
|
||||
type FileProber func(paths ...string) map[string]bool
|
||||
|
||||
// Detect resolves the distro, package manager, and init system of a
|
||||
// remote host from its /etc/os-release contents and a path prober.
|
||||
// Pure and deterministic: no command execution happens here.
|
||||
func Detect(osRelease string, probe FileProber) Detection {
|
||||
d := parseOSRelease(osRelease)
|
||||
d.PackageManager = packageManagerFor(d.ID)
|
||||
d.InitSystem = initFor(d.ID, probe)
|
||||
return d
|
||||
}
|
||||
|
||||
// parseOSRelease extracts the fields the installer cares about from the
|
||||
// /etc/os-release text. Unknown keys are ignored.
|
||||
func parseOSRelease(data string) Detection {
|
||||
var d Detection
|
||||
for _, line := range strings.Split(data, "\n") {
|
||||
line = strings.TrimSpace(line)
|
||||
if line == "" || strings.HasPrefix(line, "#") {
|
||||
continue
|
||||
}
|
||||
key, value, ok := strings.Cut(line, "=")
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
// os-release requires KEY=VALUE with no whitespace around the
|
||||
// equals sign. A spaced assignment is a malformed line and must
|
||||
// not be interpreted.
|
||||
if strings.ContainsAny(key, " \t") {
|
||||
continue
|
||||
}
|
||||
value = unquoteOSValue(value)
|
||||
switch strings.TrimSpace(key) {
|
||||
case "ID":
|
||||
d.ID = strings.ToLower(strings.TrimSpace(value))
|
||||
case "NAME":
|
||||
d.Name = strings.TrimSpace(value)
|
||||
case "VERSION_ID":
|
||||
d.VersionID = strings.TrimSpace(value)
|
||||
}
|
||||
}
|
||||
d.Distro = distroForID(d.ID)
|
||||
return d
|
||||
}
|
||||
|
||||
// unquoteOSValue strips the surrounding quotes os-release permits and
|
||||
// unescapes the two escapes the spec defines. Unquoted values pass
|
||||
// through unchanged.
|
||||
func unquoteOSValue(value string) string {
|
||||
value = strings.TrimSpace(value)
|
||||
if len(value) >= 2 && (value[0] == '"' || value[0] == '\'') && value[len(value)-1] == value[0] {
|
||||
value = value[1 : len(value)-1]
|
||||
}
|
||||
value = strings.ReplaceAll(value, `\"`, `"`)
|
||||
value = strings.ReplaceAll(value, `\'`, `'`)
|
||||
value = strings.ReplaceAll(value, `\\`, `\`)
|
||||
value = strings.ReplaceAll(value, `\$`, `$`)
|
||||
value = strings.ReplaceAll(value, "`", "")
|
||||
return value
|
||||
}
|
||||
|
||||
// distroForID maps an os-release ID to a supported distro family.
|
||||
func distroForID(id string) Distro {
|
||||
switch id {
|
||||
case "alpine":
|
||||
return DistroAlpine
|
||||
case "debian":
|
||||
return DistroDebian
|
||||
case "ubuntu", "linuxmint", "elementary":
|
||||
return DistroUbuntu
|
||||
case "arch", "archarm", "manjaro", "endeavouros":
|
||||
return DistroArch
|
||||
case "centos":
|
||||
return DistroCentOS
|
||||
case "fedora":
|
||||
return DistroFedora
|
||||
case "rocky":
|
||||
return DistroRocky
|
||||
case "almalinux":
|
||||
return DistroAlma
|
||||
case "rhel", "redhat":
|
||||
return DistroRHEL
|
||||
default:
|
||||
return DistroUnknown
|
||||
}
|
||||
}
|
||||
|
||||
// packageManagerFor maps an os-release ID to its package manager.
|
||||
// CentOS-family versions differ (dnf on 8+, yum on 7); the planner
|
||||
// defaults to dnf and the CentOS work package will refine it.
|
||||
func packageManagerFor(id string) PackageManager {
|
||||
switch distroForID(id) {
|
||||
case DistroAlpine:
|
||||
return PkgApk
|
||||
case DistroDebian, DistroUbuntu:
|
||||
return PkgApt
|
||||
case DistroArch:
|
||||
return PkgPacman
|
||||
case DistroCentOS, DistroFedora, DistroRocky, DistroAlma, DistroRHEL:
|
||||
return PkgDnf
|
||||
default:
|
||||
return PkgUnknown
|
||||
}
|
||||
}
|
||||
|
||||
// initFor infers the init system from well-known filesystem markers. In
|
||||
// a container the PID 1 command is not a reliable signal (sshd or the
|
||||
// runtime command runs first), so the installer uses the packaging
|
||||
// markers systemd and OpenRC leave behind.
|
||||
func initFor(_ string, probe FileProber) InitSystem {
|
||||
if probe == nil {
|
||||
return InitUnknown
|
||||
}
|
||||
present := probe(
|
||||
"/run/systemd/system",
|
||||
"/usr/lib/systemd/system",
|
||||
"/etc/systemd/system",
|
||||
"/sbin/openrc",
|
||||
"/etc/init.d",
|
||||
)
|
||||
if present["/run/systemd/system"] || present["/usr/lib/systemd/system"] || present["/etc/systemd/system"] {
|
||||
return InitSystemd
|
||||
}
|
||||
if present["/sbin/openrc"] || present["/etc/init.d"] {
|
||||
return InitOpenRC
|
||||
}
|
||||
return InitUnknown
|
||||
}
|
||||
|
||||
// Summarize returns a single-line, operator-readable description of the
|
||||
// detection result.
|
||||
func (d Detection) Summarize() string {
|
||||
return fmt.Sprintf("%s (id=%s version=%s, pkg=%s, init=%s)",
|
||||
displayName(d), d.ID, displayValue(d.VersionID),
|
||||
displayValue(string(d.PackageManager)), displayValue(string(d.InitSystem)))
|
||||
}
|
||||
|
||||
func displayName(d Detection) string {
|
||||
if d.Name != "" {
|
||||
return d.Name
|
||||
}
|
||||
if d.ID != "" {
|
||||
return d.ID
|
||||
}
|
||||
return string(d.Distro)
|
||||
}
|
||||
|
||||
func displayValue(v string) string {
|
||||
if v == "" {
|
||||
return "unknown"
|
||||
}
|
||||
return v
|
||||
}
|
||||
120
internal/sshinstall/detect_test.go
Обычный файл
120
internal/sshinstall/detect_test.go
Обычный файл
@@ -0,0 +1,120 @@
|
||||
package sshinstall
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// prober returns a FileProber backed by a fixed path set.
|
||||
func prober(exist ...string) FileProber {
|
||||
want := make(map[string]bool, len(exist))
|
||||
for _, p := range exist {
|
||||
want[p] = true
|
||||
}
|
||||
return func(paths ...string) map[string]bool {
|
||||
out := make(map[string]bool, len(paths))
|
||||
for _, p := range paths {
|
||||
out[p] = want[p]
|
||||
}
|
||||
return out
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseOSReleaseQuoting(t *testing.T) {
|
||||
d := parseOSRelease(`NAME="Ubuntu 24.04 LTS"
|
||||
VERSION_ID="24.04"
|
||||
ID=ubuntu
|
||||
ID_LIKE=debian
|
||||
PRETTY_NAME="Ubuntu 24.04 LTS"
|
||||
`)
|
||||
if d.ID != "ubuntu" || d.Name != "Ubuntu 24.04 LTS" || d.VersionID != "24.04" {
|
||||
t.Fatalf("parseOSRelease = %+v", d)
|
||||
}
|
||||
if d.Distro != DistroUbuntu {
|
||||
t.Fatalf("distro = %q, want ubuntu", d.Distro)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseOSReleaseSingleQuotedAndEmpty(t *testing.T) {
|
||||
d := parseOSRelease("# comment\nID='alpine'\nNAME=Alpine\n\n")
|
||||
if d.ID != "alpine" || d.Name != "Alpine" {
|
||||
t.Fatalf("parseOSRelease = %+v", d)
|
||||
}
|
||||
if d.Distro != DistroAlpine {
|
||||
t.Fatalf("distro = %q, want alpine", d.Distro)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseOSReleaseGarbage(t *testing.T) {
|
||||
d := parseOSRelease("not an os-release file\nID = spaced\nFOO=bar\n")
|
||||
if d.ID != "" {
|
||||
t.Fatalf("ID = %q, want empty", d.ID)
|
||||
}
|
||||
if d.Distro != DistroUnknown {
|
||||
t.Fatalf("distro = %q, want unknown", d.Distro)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPackageManagerFor(t *testing.T) {
|
||||
cases := map[string]PackageManager{
|
||||
"alpine": PkgApk,
|
||||
"debian": PkgApt,
|
||||
"ubuntu": PkgApt,
|
||||
"arch": PkgPacman,
|
||||
"centos": PkgDnf,
|
||||
"fedora": PkgDnf,
|
||||
"rocky": PkgDnf,
|
||||
"almalinux": PkgDnf,
|
||||
"rhel": PkgDnf,
|
||||
"nonsense": PkgUnknown,
|
||||
"": PkgUnknown,
|
||||
}
|
||||
for id, want := range cases {
|
||||
if got := packageManagerFor(id); got != want {
|
||||
t.Fatalf("packageManagerFor(%q) = %q, want %q", id, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestDetectInitMarkers(t *testing.T) {
|
||||
osRelease := "ID=arch\nNAME=Arch Linux\n"
|
||||
|
||||
if got := Detect(osRelease, prober("/usr/lib/systemd/system")); got.InitSystem != InitSystemd {
|
||||
t.Fatalf("arch systemd markers not detected: %+v", got)
|
||||
}
|
||||
if got := Detect(osRelease, nil).InitSystem; got != InitUnknown {
|
||||
t.Fatalf("nil prober should yield unknown init, got %q", got)
|
||||
}
|
||||
|
||||
alpine := Detect("ID=alpine\n", prober("/sbin/openrc", "/etc/init.d"))
|
||||
if alpine.InitSystem != InitOpenRC {
|
||||
t.Fatalf("alpine openrc markers not detected: %+v", alpine)
|
||||
}
|
||||
if alpine.PackageManager != PkgApk {
|
||||
t.Fatalf("alpine pkg = %q, want apk", alpine.PackageManager)
|
||||
}
|
||||
|
||||
ubuntu := Detect("ID=ubuntu\n", prober("/run/systemd/system"))
|
||||
if ubuntu.InitSystem != InitSystemd || ubuntu.PackageManager != PkgApt {
|
||||
t.Fatalf("ubuntu detection = %+v", ubuntu)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDetectSystemdWinsOverOpenRC(t *testing.T) {
|
||||
// A host that carries both markers must be reported as systemd so a
|
||||
// systemd distro running a containerized openrc stub is not misplanned.
|
||||
d := Detect("ID=ubuntu\n", prober("/usr/lib/systemd/system", "/sbin/openrc"))
|
||||
if d.InitSystem != InitSystemd {
|
||||
t.Fatalf("init = %q, want systemd", d.InitSystem)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDetectionSummarize(t *testing.T) {
|
||||
d := Detect("ID=alpine\nNAME=Alpine Linux\n", prober("/sbin/openrc"))
|
||||
s := d.Summarize()
|
||||
for _, want := range []string{"Alpine Linux", "id=alpine", "pkg=apk", "init=openrc"} {
|
||||
if !strings.Contains(s, want) {
|
||||
t.Fatalf("Summarize() = %q missing %q", s, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
298
internal/sshinstall/plan.go
Обычный файл
298
internal/sshinstall/plan.go
Обычный файл
@@ -0,0 +1,298 @@
|
||||
package sshinstall
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/url"
|
||||
"regexp"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// DefaultGoVersion is the pinned Go toolchain version the source
|
||||
// installer plans by default. It is overridable through install options
|
||||
// when a host needs a different toolchain.
|
||||
const DefaultGoVersion = "1.26.0"
|
||||
|
||||
// DefaultRepo is the publicly readable worker repository. It requires no
|
||||
// source credential and is the plan's default clone URL.
|
||||
const DefaultRepo = "https://rocketgit.ru/rsmon/worker.git"
|
||||
|
||||
var (
|
||||
// goVersionPattern bounds Go toolchain version strings that are
|
||||
// interpolated into remote shell commands and download URLs.
|
||||
goVersionPattern = regexp.MustCompile(`^[0-9][0-9A-Za-z._-]*$`)
|
||||
// goArchPattern bounds Go download archive suffixes (e.g. "amd64")
|
||||
// that are interpolated into remote shell commands and URLs.
|
||||
goArchPattern = regexp.MustCompile(`^[0-9A-Za-z][0-9A-Za-z_-]*$`)
|
||||
)
|
||||
|
||||
// ValidGoVersion reports whether a Go toolchain version uses only safe
|
||||
// characters (digits, letters, dots, dashes, underscores) and starts
|
||||
// with a digit. Versions are embedded in remote shell commands and
|
||||
// download URLs, so the charset is the injection boundary.
|
||||
func ValidGoVersion(v string) bool { return goVersionPattern.MatchString(v) }
|
||||
|
||||
// ValidGoArch reports whether a Go download archive suffix uses only
|
||||
// safe characters. Suffixes are embedded in remote shell commands and
|
||||
// download URLs, so the charset is the injection boundary.
|
||||
func ValidGoArch(a string) bool { return goArchPattern.MatchString(a) }
|
||||
|
||||
// Toolchain is a pinned, checksum-verified Go toolchain download for a
|
||||
// remote Linux architecture. The SHA-256 is baked for the default
|
||||
// toolchain version so planning never needs the network.
|
||||
type Toolchain struct {
|
||||
Version string // e.g. "1.26.0"
|
||||
Arch string // download archive suffix, e.g. "linux-amd64"
|
||||
URL string // direct download URL
|
||||
SHA256 string // published SHA-256 (64 lowercase hex)
|
||||
}
|
||||
|
||||
// toolchainSHA pins the official go.dev SHA-256 checksums for the
|
||||
// default Go version per Linux archive. Sources: https://go.dev/dl/
|
||||
// (?mode=json) published sums for DefaultGoVersion. Keep this in sync
|
||||
// with DefaultGoVersion.
|
||||
var toolchainSHA = map[string]string{
|
||||
"linux-386": "35e2ec7a7ae6905a1fae5459197b70e3fcbc5e0a786a7d6ba8e49bcd38ad2e26",
|
||||
"linux-amd64": "aac1b08a0fb0c4e0a7c1555beb7b59180b05dfc5a3d62e40e9de90cd42f88235",
|
||||
"linux-arm64": "bd03b743eb6eb4193ea3c3fd3956546bf0e3ca5b7076c8226334afe6b75704cd",
|
||||
"linux-armv6l": "3f6b48d96f0d8dff77e4625aa179e0449f6bbe79b6986bfa711c2cfc1257ebd8",
|
||||
"linux-loong64": "33947cd7686f1cd5f097d2a5a30427a4ade114ea00b7570c85a2abf1af3d0507",
|
||||
"linux-mips": "a4ece61d4bac43b6983fde2c6b9cfc1af7f0d5d6a073219583d4e93b11559c25",
|
||||
"linux-mips64": "197c2e97fa9ec1ad05998e0982d1a1ae761980df154424e5f29f3912e9ea4e5e",
|
||||
"linux-mips64le": "61c52b4ab0dceae29f10df29045483596c3f06810c9b511e8336a97428a95a1b",
|
||||
"linux-mipsle": "b3a13cc5a5f9250b02cf4ba19914c90c7034e68a5ccb9affa5198aadbcedac9a",
|
||||
"linux-ppc64": "ef7232a49101d163a93bac34d03bfbc4fb18f75d7526d77ac307e16d9d83c300",
|
||||
"linux-ppc64le": "3066b2284b554da76cf664d217490792ba6f292ec0fc20bf9615e173cc0d2800",
|
||||
"linux-riscv64": "ab9226ecddda0f682365c949114b653a66c2e9330e7b8d3edea80858437d2ff2",
|
||||
"linux-s390x": "d62137f11530b97f3503453ad7d9e570af070770599fb8054f4e8cd0e905a453",
|
||||
}
|
||||
|
||||
// GoArch maps a remote `uname -m` value to the Go download archive
|
||||
// suffix used by go.dev. Every suffix it can return must have a pinned
|
||||
// checksum in toolchainSHA; the table-consistency test enforces that.
|
||||
// Unknown values error.
|
||||
func GoArch(unameM string) (string, error) {
|
||||
switch strings.ToLower(strings.TrimSpace(unameM)) {
|
||||
case "x86_64", "amd64":
|
||||
return "amd64", nil
|
||||
case "aarch64", "arm64":
|
||||
return "arm64", nil
|
||||
case "armv6l", "armv7l":
|
||||
return "armv6l", nil
|
||||
case "i386", "i486", "i586", "i686", "386":
|
||||
return "386", nil
|
||||
case "loongarch64":
|
||||
return "loong64", nil
|
||||
case "mips":
|
||||
return "mips", nil
|
||||
case "mipsel":
|
||||
return "mipsle", nil
|
||||
case "mips64":
|
||||
return "mips64", nil
|
||||
case "mips64el":
|
||||
return "mips64le", nil
|
||||
case "ppc64":
|
||||
return "ppc64", nil
|
||||
case "ppc64le":
|
||||
return "ppc64le", nil
|
||||
case "riscv64":
|
||||
return "riscv64", nil
|
||||
case "s390x":
|
||||
return "s390x", nil
|
||||
default:
|
||||
return "", fmt.Errorf("unsupported machine architecture %q", unameM)
|
||||
}
|
||||
}
|
||||
|
||||
// ToolchainFor returns the pinned, checksum-verified Go toolchain for a
|
||||
// remote Linux architecture. A non-default version has no baked
|
||||
// checksum yet and must be resolved through a checksum source by the
|
||||
// executor work package.
|
||||
func ToolchainFor(goarch, version string) (Toolchain, error) {
|
||||
if version == "" {
|
||||
version = DefaultGoVersion
|
||||
}
|
||||
suffix := "linux-" + goarch
|
||||
t := Toolchain{
|
||||
Version: version,
|
||||
Arch: suffix,
|
||||
URL: "https://go.dev/dl/go" + version + "." + suffix + ".tar.gz",
|
||||
}
|
||||
if version != DefaultGoVersion {
|
||||
return t, fmt.Errorf("no baked checksum for Go %s; only %s is pinned (resolve %s via the checksum source)",
|
||||
version, DefaultGoVersion, suffix)
|
||||
}
|
||||
sha, ok := toolchainSHA[suffix]
|
||||
if !ok {
|
||||
return Toolchain{}, fmt.Errorf("no pinned Go %s toolchain for %s", version, suffix)
|
||||
}
|
||||
t.SHA256 = sha
|
||||
return t, nil
|
||||
}
|
||||
|
||||
// SourceOptions are the operator-configurable knobs that the source
|
||||
// plan is resolved against. Every field is optional; empty values fall
|
||||
// back to the pinned defaults.
|
||||
type SourceOptions struct {
|
||||
Repo string // clone URL; default DefaultRepo
|
||||
Branch string // branch to build; empty means "the remote default branch"
|
||||
GoVersion string // default DefaultGoVersion
|
||||
GoArch string // go archive suffix; when empty, derived from UnameM
|
||||
UnameM string // remote `uname -m` output; required unless GoArch set
|
||||
BuildDir string // remote clone/build directory
|
||||
GoModuleProxy string // GOPROXY override; empty keeps the Go default
|
||||
}
|
||||
|
||||
// SourcePlan is the pure, resolved source-install plan. Producing it
|
||||
// never touches the network or the remote host; the executor work
|
||||
// package turns it into remote commands.
|
||||
type SourcePlan struct {
|
||||
Repo string
|
||||
Branch string
|
||||
BuildDir string
|
||||
GoModuleProxy string
|
||||
Toolchain Toolchain
|
||||
Packages []string // packages to install via the package manager
|
||||
InitSystem InitSystem
|
||||
}
|
||||
|
||||
// PlanSource resolves a source-install plan for a detected host. It
|
||||
// returns an error before any remote mutation could happen when the
|
||||
// target cannot be planned (unknown distro, unsupported architecture,
|
||||
// malformed repository).
|
||||
func PlanSource(d Detection, opts SourceOptions) (SourcePlan, error) {
|
||||
if d.PackageManager == PkgUnknown {
|
||||
return SourcePlan{}, fmt.Errorf("unsupported distro %q: no package manager", d.ID)
|
||||
}
|
||||
goarch := strings.TrimSpace(opts.GoArch)
|
||||
if goarch != "" && !ValidGoArch(goarch) {
|
||||
return SourcePlan{}, fmt.Errorf("invalid Go architecture %q: only letters, digits, dashes, and underscores are allowed", goarch)
|
||||
}
|
||||
if goarch == "" {
|
||||
var err error
|
||||
goarch, err = GoArch(opts.UnameM)
|
||||
if err != nil {
|
||||
return SourcePlan{}, err
|
||||
}
|
||||
}
|
||||
if version := strings.TrimSpace(opts.GoVersion); version != "" && !ValidGoVersion(version) {
|
||||
return SourcePlan{}, fmt.Errorf("invalid Go version %q: only digits, letters, dots, dashes, and underscores are allowed", version)
|
||||
}
|
||||
toolchain, err := ToolchainFor(goarch, opts.GoVersion)
|
||||
if err != nil {
|
||||
return SourcePlan{}, err
|
||||
}
|
||||
repo := strings.TrimSpace(opts.Repo)
|
||||
if repo == "" {
|
||||
repo = DefaultRepo
|
||||
}
|
||||
if err := validateRepoURL(repo); err != nil {
|
||||
return SourcePlan{}, err
|
||||
}
|
||||
branch := strings.TrimSpace(opts.Branch)
|
||||
// An empty branch means "build the remote's default branch" (the
|
||||
// public repo currently publishes master). The executor resolves and
|
||||
// records the remote default; a non-empty branch is pinned and must
|
||||
// exist on the remote.
|
||||
buildDir := strings.TrimSpace(opts.BuildDir)
|
||||
if buildDir == "" {
|
||||
buildDir = "/opt/rsmon-worker-src"
|
||||
}
|
||||
if !strings.HasPrefix(buildDir, "/") {
|
||||
return SourcePlan{}, fmt.Errorf("build directory must be absolute, got %q", buildDir)
|
||||
}
|
||||
return SourcePlan{
|
||||
Repo: repo,
|
||||
Branch: branch,
|
||||
BuildDir: buildDir,
|
||||
GoModuleProxy: strings.TrimSpace(opts.GoModuleProxy),
|
||||
Toolchain: toolchain,
|
||||
Packages: packagePrereqs(d.PackageManager),
|
||||
InitSystem: d.InitSystem,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// packagePrereqs returns the minimal package set the source installer
|
||||
// plans to install for a package manager: git, CA certificates, and
|
||||
// download/archive tools. It never plans a C compiler or build-essential
|
||||
// because the worker builds with CGO disabled.
|
||||
func packagePrereqs(pkg PackageManager) []string {
|
||||
switch pkg {
|
||||
case PkgApk:
|
||||
return []string{"git", "ca-certificates", "curl", "tar", "gzip"}
|
||||
case PkgApt:
|
||||
return []string{"git", "ca-certificates", "curl", "tar", "gzip"}
|
||||
case PkgPacman:
|
||||
return []string{"git", "ca-certificates", "curl", "tar", "gzip"}
|
||||
case PkgDnf, PkgYum:
|
||||
return []string{"git", "ca-certificates", "curl", "tar", "gzip"}
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// ValidateRepoURL rejects repository references that could smuggle a
|
||||
// command or a non-remote scheme into the clone step. Only https is
|
||||
// accepted (the default repository is https), and userinfo such as
|
||||
// `user:pass@host` is rejected so credentials can never reach the remote
|
||||
// clone command or the clone's config. An empty value is accepted here
|
||||
// (it falls back to the default repository when planning).
|
||||
func ValidateRepoURL(repo string) error {
|
||||
if repo == "" {
|
||||
return nil
|
||||
}
|
||||
return validateRepoURL(repo)
|
||||
}
|
||||
|
||||
func validateRepoURL(repo string) error {
|
||||
if strings.ContainsAny(repo, "\r\n\t ") {
|
||||
return fmt.Errorf("repository URL %q contains whitespace", repo)
|
||||
}
|
||||
u, err := url.Parse(repo)
|
||||
if err != nil || u.Host == "" {
|
||||
return fmt.Errorf("repository URL %q is not an absolute clone URL", repo)
|
||||
}
|
||||
if u.User != nil {
|
||||
return fmt.Errorf("repository URL %q must not contain userinfo", repo)
|
||||
}
|
||||
if u.Scheme != "https" {
|
||||
return fmt.Errorf("repository URL %q must use the https scheme", repo)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// StepKind identifies one ordered remote step the executor will run.
|
||||
type StepKind string
|
||||
|
||||
const (
|
||||
StepInstallPackages StepKind = "install-packages"
|
||||
StepInstallToolchain StepKind = "install-toolchain"
|
||||
StepCloneSource StepKind = "clone-source"
|
||||
StepCheckoutBranch StepKind = "checkout-branch"
|
||||
StepBuildWorker StepKind = "build-worker"
|
||||
StepInstallService StepKind = "install-service"
|
||||
)
|
||||
|
||||
// Step is one ordered, pure planning step. The executor maps each step
|
||||
// to remote commands; planning does not execute anything.
|
||||
type Step struct {
|
||||
Kind StepKind
|
||||
Detail string
|
||||
Packages []string // only for StepInstallPackages
|
||||
}
|
||||
|
||||
// Steps returns the ordered source-install plan as stable, reviewable
|
||||
// steps. It is the contract the executor work package implements.
|
||||
func (p SourcePlan) Steps() []Step {
|
||||
checkoutDetail := "check out branch " + p.Branch + " and record the resolved commit"
|
||||
if p.Branch == "" {
|
||||
checkoutDetail = "check out the remote default branch and record the resolved commit"
|
||||
}
|
||||
return []Step{
|
||||
{Kind: StepInstallPackages, Detail: "install minimal build prerequisites", Packages: p.Packages},
|
||||
{Kind: StepInstallToolchain, Detail: "install pinned Go " + p.Toolchain.Version + " (" + p.Toolchain.Arch + ") and verify SHA-256"},
|
||||
{Kind: StepCloneSource, Detail: "clone " + p.Repo + " into " + p.BuildDir},
|
||||
{Kind: StepCheckoutBranch, Detail: checkoutDetail},
|
||||
{Kind: StepBuildWorker, Detail: "build the worker binary with CGO_ENABLED=0 and trimpath"},
|
||||
{Kind: StepInstallService, Detail: "atomically install the binary, env, data dir, and " + string(p.InitSystem) + " service definition"},
|
||||
}
|
||||
}
|
||||
274
internal/sshinstall/plan_test.go
Обычный файл
274
internal/sshinstall/plan_test.go
Обычный файл
@@ -0,0 +1,274 @@
|
||||
package sshinstall
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestGoArch(t *testing.T) {
|
||||
cases := map[string]string{
|
||||
"x86_64": "amd64",
|
||||
"X86_64": "amd64",
|
||||
"amd64": "amd64",
|
||||
"aarch64": "arm64",
|
||||
"arm64": "arm64",
|
||||
"armv7l": "armv6l",
|
||||
"armv6l": "armv6l",
|
||||
"i686": "386",
|
||||
"i386": "386",
|
||||
"loongarch64": "loong64",
|
||||
"mips": "mips",
|
||||
"mipsel": "mipsle",
|
||||
"mips64": "mips64",
|
||||
"mips64el": "mips64le",
|
||||
"ppc64": "ppc64",
|
||||
"ppc64le": "ppc64le",
|
||||
"riscv64": "riscv64",
|
||||
"s390x": "s390x",
|
||||
}
|
||||
for in, want := range cases {
|
||||
got, err := GoArch(in)
|
||||
if err != nil || got != want {
|
||||
t.Fatalf("GoArch(%q) = %q, %v; want %q", in, got, err, want)
|
||||
}
|
||||
}
|
||||
for _, bad := range []string{"", "sparc", "x86", "mips64el-le"} {
|
||||
if _, err := GoArch(bad); err == nil {
|
||||
t.Fatalf("GoArch(%q) succeeded", bad)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestToolchainTableConsistentWithGoArch keeps GoArch and toolchainSHA in
|
||||
// sync: every `uname -m` mapping must resolve to a pinned checksum, and
|
||||
// every pinned checksum must be reachable through some `uname -m` value.
|
||||
func TestToolchainTableConsistentWithGoArch(t *testing.T) {
|
||||
// unameExamples maps a Go archive suffix to a real `uname -m` value
|
||||
// that GoArch accepts for it.
|
||||
unameExamples := map[string]string{
|
||||
"amd64": "x86_64",
|
||||
"arm64": "aarch64",
|
||||
"armv6l": "armv7l",
|
||||
"386": "i686",
|
||||
"loong64": "loongarch64",
|
||||
"mips": "mips",
|
||||
"mipsle": "mipsel",
|
||||
"mips64": "mips64",
|
||||
"mips64le": "mips64el",
|
||||
"ppc64": "ppc64",
|
||||
"ppc64le": "ppc64le",
|
||||
"riscv64": "riscv64",
|
||||
"s390x": "s390x",
|
||||
}
|
||||
for archSuffix, unameValue := range unameExamples {
|
||||
tc, err := ToolchainFor(archSuffix, DefaultGoVersion)
|
||||
if err != nil {
|
||||
t.Fatalf("GoArch(%q) = %q has no pinned checksum: %v", unameValue, archSuffix, err)
|
||||
}
|
||||
if tc.SHA256 == "" || tc.Arch != "linux-"+archSuffix {
|
||||
t.Fatalf("ToolchainFor(%q) = %+v", archSuffix, tc)
|
||||
}
|
||||
}
|
||||
for suffix := range toolchainSHA {
|
||||
arch := strings.TrimPrefix(suffix, "linux-")
|
||||
if _, ok := unameExamples[arch]; !ok {
|
||||
t.Fatalf("checksum %q is not reachable through any GoArch `uname -m` mapping", suffix)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestToolchainFor(t *testing.T) {
|
||||
tc, err := ToolchainFor("amd64", "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if tc.Version != DefaultGoVersion {
|
||||
t.Fatalf("version = %q, want %q", tc.Version, DefaultGoVersion)
|
||||
}
|
||||
if tc.Arch != "linux-amd64" {
|
||||
t.Fatalf("arch = %q, want linux-amd64", tc.Arch)
|
||||
}
|
||||
if tc.URL != "https://go.dev/dl/go1.26.0.linux-amd64.tar.gz" {
|
||||
t.Fatalf("url = %q", tc.URL)
|
||||
}
|
||||
if len(tc.SHA256) != 64 {
|
||||
t.Fatalf("sha = %q, want 64 hex chars", tc.SHA256)
|
||||
}
|
||||
for _, c := range tc.SHA256 {
|
||||
hexDigit := c >= '0' && c <= '9' || c >= 'a' && c <= 'f'
|
||||
if !hexDigit {
|
||||
t.Fatalf("sha %q contains non-lowercase-hex char", tc.SHA256)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestToolchainForArm64Pinned(t *testing.T) {
|
||||
tc, err := ToolchainFor("arm64", DefaultGoVersion)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if tc.SHA256 == "" || tc.Arch != "linux-arm64" {
|
||||
t.Fatalf("arm64 toolchain not pinned: %+v", tc)
|
||||
}
|
||||
}
|
||||
|
||||
func TestToolchainForRejectsUnpinnedVersion(t *testing.T) {
|
||||
if _, err := ToolchainFor("amd64", "1.27.0"); err == nil {
|
||||
t.Fatal("unpinned Go version accepted")
|
||||
}
|
||||
if _, err := ToolchainFor("sparc", ""); err == nil {
|
||||
t.Fatal("unknown architecture accepted")
|
||||
}
|
||||
}
|
||||
|
||||
func TestPlanSourceAlpine(t *testing.T) {
|
||||
d := Detect("ID=alpine\n", prober("/sbin/openrc"))
|
||||
p, err := PlanSource(d, SourceOptions{UnameM: "x86_64"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if p.Repo != DefaultRepo || p.Branch != "" {
|
||||
t.Fatalf("plan defaults wrong: %+v", p)
|
||||
}
|
||||
if p.Toolchain.Arch != "linux-amd64" {
|
||||
t.Fatalf("toolchain = %+v", p.Toolchain)
|
||||
}
|
||||
if len(p.Packages) != 5 || p.Packages[0] != "git" {
|
||||
t.Fatalf("alpine packages = %v", p.Packages)
|
||||
}
|
||||
if p.InitSystem != InitOpenRC {
|
||||
t.Fatalf("init = %q, want openrc", p.InitSystem)
|
||||
}
|
||||
steps := p.Steps()
|
||||
if len(steps) != 6 {
|
||||
t.Fatalf("steps = %d, want 6", len(steps))
|
||||
}
|
||||
if steps[0].Kind != StepInstallPackages || steps[1].Kind != StepInstallToolchain {
|
||||
t.Fatalf("step order wrong: %+v", steps)
|
||||
}
|
||||
if !strings.Contains(steps[1].Detail, "Go 1.26.0") || !strings.Contains(steps[1].Detail, "linux-amd64") {
|
||||
t.Fatalf("toolchain step detail = %q", steps[1].Detail)
|
||||
}
|
||||
if !strings.Contains(steps[5].Detail, "openrc") {
|
||||
t.Fatalf("install step detail = %q", steps[5].Detail)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPlanSourceUbuntuArch(t *testing.T) {
|
||||
ubuntu := Detect("ID=ubuntu\n", prober("/usr/lib/systemd/system"))
|
||||
p, err := PlanSource(ubuntu, SourceOptions{UnameM: "aarch64"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if p.Toolchain.Arch != "linux-arm64" || p.InitSystem != InitSystemd {
|
||||
t.Fatalf("ubuntu plan = %+v", p)
|
||||
}
|
||||
|
||||
arch := Detect("ID=arch\n", prober("/usr/lib/systemd/system"))
|
||||
p, err = PlanSource(arch, SourceOptions{UnameM: "x86_64"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if p.Packages[0] != "git" || p.InitSystem != InitSystemd {
|
||||
t.Fatalf("arch plan = %+v", p)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPlanSourceExplicitOverrides(t *testing.T) {
|
||||
d := Detect("ID=ubuntu\n", nil)
|
||||
p, err := PlanSource(d, SourceOptions{
|
||||
Repo: "https://example.test/worker.git",
|
||||
Branch: "release-1.0",
|
||||
GoArch: "arm64",
|
||||
BuildDir: "/srv/rsmon",
|
||||
GoModuleProxy: "https://proxy.golang.org,direct",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if p.Repo != "https://example.test/worker.git" || p.Branch != "release-1.0" ||
|
||||
p.BuildDir != "/srv/rsmon" || p.GoModuleProxy != "https://proxy.golang.org,direct" {
|
||||
t.Fatalf("overrides not applied: %+v", p)
|
||||
}
|
||||
if p.Toolchain.Arch != "linux-arm64" {
|
||||
t.Fatalf("toolchain = %+v", p.Toolchain)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPlanSourceErrors(t *testing.T) {
|
||||
unknown := Detect("ID=weirdos\n", nil)
|
||||
if _, err := PlanSource(unknown, SourceOptions{UnameM: "x86_64"}); err == nil {
|
||||
t.Fatal("unknown distro planned")
|
||||
}
|
||||
|
||||
ubuntu := Detect("ID=ubuntu\n", nil)
|
||||
if _, err := PlanSource(ubuntu, SourceOptions{}); err == nil {
|
||||
t.Fatal("missing architecture planned")
|
||||
}
|
||||
for _, repo := range []string{"file:///tmp/worker.git", "not-a-url", "https://ex ample/x"} {
|
||||
if _, err := PlanSource(ubuntu, SourceOptions{UnameM: "x86_64", Repo: repo}); err == nil {
|
||||
t.Fatalf("invalid repo %q planned", repo)
|
||||
}
|
||||
}
|
||||
if _, err := PlanSource(ubuntu, SourceOptions{UnameM: "x86_64", BuildDir: "relative"}); err == nil {
|
||||
t.Fatal("relative build dir planned")
|
||||
}
|
||||
}
|
||||
|
||||
func TestPackagePrereqsNeverIncludeCompiler(t *testing.T) {
|
||||
for _, pkg := range []PackageManager{PkgApk, PkgApt, PkgPacman, PkgDnf} {
|
||||
for _, name := range packagePrereqs(pkg) {
|
||||
switch name {
|
||||
case "build-essential", "gcc", "g++", "base-devel", "gcc-c++", "make":
|
||||
t.Fatalf("plan includes compiler package %q", name)
|
||||
}
|
||||
}
|
||||
}
|
||||
if got := packagePrereqs(PkgUnknown); len(got) != 0 {
|
||||
t.Fatalf("unknown package manager planned packages %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateRepoURLSchemes(t *testing.T) {
|
||||
for _, ok := range []string{"https://rocketgit.ru/rsmon/worker.git", "https://example.test/r"} {
|
||||
if err := validateRepoURL(ok); err != nil {
|
||||
t.Fatalf("validateRepoURL(%q): %v", ok, err)
|
||||
}
|
||||
}
|
||||
for _, bad := range []string{
|
||||
"ssh://h@x/r", "s3://bucket/key", "x y", "", "http://x/y", "git://example.test/r",
|
||||
"https://user:pass@example.test/r", "https://token@example.test/r", "file:///tmp/r",
|
||||
} {
|
||||
if err := validateRepoURL(bad); err == nil {
|
||||
t.Fatalf("validateRepoURL(%q) succeeded", bad)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestPlanSourceRejectsUnsafeCharset(t *testing.T) {
|
||||
ubuntu := Detect("ID=ubuntu\n", nil)
|
||||
for _, goarch := range []string{"amd64;rm", "x;rm -rf", "$(id)", "..", "a b"} {
|
||||
if _, err := PlanSource(ubuntu, SourceOptions{UnameM: "x86_64", GoArch: goarch}); err == nil {
|
||||
t.Fatalf("unsafe GoArch %q planned", goarch)
|
||||
}
|
||||
}
|
||||
for _, version := range []string{"1.26;rm", "$(id)", "1.26.0 x", "a/b"} {
|
||||
if _, err := PlanSource(ubuntu, SourceOptions{UnameM: "x86_64", GoVersion: version}); err == nil {
|
||||
t.Fatalf("unsafe GoVersion %q planned", version)
|
||||
}
|
||||
}
|
||||
if !ValidGoVersion("1.26.0") || !ValidGoArch("amd64") {
|
||||
t.Fatal("valid version/arch rejected")
|
||||
}
|
||||
}
|
||||
|
||||
func TestPlanSourceEmptyBranchSteps(t *testing.T) {
|
||||
ubuntu := Detect("ID=ubuntu\n", nil)
|
||||
p, err := PlanSource(ubuntu, SourceOptions{UnameM: "x86_64"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.Contains(p.Steps()[3].Detail, "remote default branch") {
|
||||
t.Fatalf("checkout step detail for empty branch = %q", p.Steps()[3].Detail)
|
||||
}
|
||||
}
|
||||
@@ -25,6 +25,17 @@ type RegisterResponse struct {
|
||||
WorkerID string `json:"worker_id"`
|
||||
}
|
||||
|
||||
type BootstrapRequest struct {
|
||||
WorkerID string `json:"worker_id"`
|
||||
BootstrapToken string `json:"bootstrap_token"`
|
||||
}
|
||||
type BootstrapResponse struct {
|
||||
AuthToken string `json:"auth_token"`
|
||||
WorkerID string `json:"worker_id"`
|
||||
ConfigVerificationKey string `json:"config_verification_key"`
|
||||
SigningKeyID string `json:"signing_key_id"`
|
||||
}
|
||||
|
||||
// HeartbeatRequest is sent periodically by workers to indicate liveness.
|
||||
type HeartbeatRequest struct {
|
||||
ActiveChecks int `json:"active_checks"`
|
||||
@@ -44,6 +55,7 @@ type CheckJob struct {
|
||||
URL *string `json:"url"`
|
||||
Interval int `json:"interval"`
|
||||
Settings json.RawMessage `json:"settings"` // CheckSettings JSON
|
||||
AccountID int64 `json:"account_id,omitempty"`
|
||||
}
|
||||
|
||||
// JobsResponse contains a batch of check jobs for a worker
|
||||
@@ -230,7 +242,16 @@ type WorkerInit struct {
|
||||
// the master API selfcheck. The current worker is excluded by
|
||||
// the control plane; a worker that receives an empty list treats
|
||||
// the selfcheck as a single-node decision (no peer polling).
|
||||
Peers []PeerInfo `json:"peers,omitempty"`
|
||||
Peers []PeerInfo `json:"peers,omitempty"`
|
||||
AccountID *int64 `json:"account_id,omitempty"`
|
||||
ConfigVersion int64 `json:"config_version,omitempty"`
|
||||
IssuedAt string `json:"issued_at,omitempty"`
|
||||
ExpiresAt string `json:"expires_at,omitempty"`
|
||||
CredentialSetHash string `json:"credential_set_hash,omitempty"`
|
||||
Signature string `json:"signature,omitempty"`
|
||||
SigningKeyID string `json:"signing_key_id,omitempty"`
|
||||
RotateToken string `json:"rotate_token,omitempty"`
|
||||
RotationID string `json:"rotation_id,omitempty"`
|
||||
}
|
||||
|
||||
// TaskEnvelope is the task frame the control plane sends on the worker
|
||||
@@ -324,5 +345,20 @@ type WorkerMessage struct {
|
||||
NotificationResult *NotificationResultReport `json:"notification_result,omitempty"`
|
||||
ServerMetric *ServerMetricReport `json:"server_metric,omitempty"`
|
||||
Heartbeat *HeartbeatRequest `json:"heartbeat,omitempty"`
|
||||
RotationAck *RotationAck `json:"rotation_ack,omitempty"`
|
||||
StaleLease *StaleLeaseReport `json:"stale_lease,omitempty"`
|
||||
StaleLeaseAck *StaleLeaseAck `json:"stale_lease_ack,omitempty"`
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
type RotationAck struct {
|
||||
RotationID string `json:"rotation_id"`
|
||||
}
|
||||
type StaleLeaseReport struct {
|
||||
JobID string `json:"job_id"`
|
||||
LeaseToken string `json:"lease_token"`
|
||||
}
|
||||
type StaleLeaseAck struct {
|
||||
JobID string `json:"job_id"`
|
||||
LeaseToken string `json:"lease_token"`
|
||||
}
|
||||
|
||||
78
scripts/ci/test-ssh.sh
Исполняемый файл
78
scripts/ci/test-ssh.sh
Исполняемый файл
@@ -0,0 +1,78 @@
|
||||
#!/usr/bin/env bash
|
||||
# CI entrypoint for the source-install Docker/OpenSSH E2E matrix
|
||||
# (work package 5, docs/source-installation.md).
|
||||
#
|
||||
# Runs `make test-ssh` (Alpine/Ubuntu/Arch real OpenSSH fixtures over the
|
||||
# public network) inside a Docker-capable CI job. It:
|
||||
# - fails fast with a clear message when Docker is unreachable, because the
|
||||
# harness dials fixture SSH ports published on the daemon's loopback;
|
||||
# - runs the full fixture matrix with make's 60m go-test timeout;
|
||||
# - cleans up every leftover test container/network/fixture-image tag on
|
||||
# success AND failure, so an aborted job never leaks test material on the
|
||||
# runner.
|
||||
#
|
||||
# It never deletes a shared base image: only rsmon-worker-test-* resources and
|
||||
# the per-instance rsmon-worker-test/<fixture>-<suffix>:local tags the harness
|
||||
# owns are removed.
|
||||
set -euo pipefail
|
||||
|
||||
# cleanup removes only resources the harness owns. Runs on every exit path.
|
||||
cleanup() {
|
||||
# Containers, then networks (which need no attached containers). Filters
|
||||
# are anchored to the harness's own prefix so a resource that merely
|
||||
# contains the substring is never touched.
|
||||
docker ps -aq --filter "name=^rsmon-worker-test-" 2>/dev/null \
|
||||
| xargs -r docker rm -f >/dev/null 2>&1 || true
|
||||
docker network ls -q --filter "name=^rsmon-worker-test-" 2>/dev/null \
|
||||
| xargs -r docker network rm >/dev/null 2>&1 || true
|
||||
# Per-instance fixture image tags only; never a shared base image.
|
||||
docker images -q --filter "reference=rsmon-worker-test/*" 2>/dev/null \
|
||||
| xargs -r docker image rm -f >/dev/null 2>&1 || true
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
if ! command -v docker >/dev/null 2>&1; then
|
||||
echo "test-ssh CI: docker CLI not found; this job needs a Docker-capable runner" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! docker info >/dev/null 2>&1; then
|
||||
echo "test-ssh CI: Docker daemon is not reachable; this job needs a working Docker daemon" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "test-ssh CI: Docker $(docker version --format '{{.Server.Version}}' 2>/dev/null || echo n/a) ready"
|
||||
|
||||
# The harness publishes fixture SSH ports to the daemon's 127.0.0.1 and
|
||||
# dials them from the test process, so the job must share the daemon's
|
||||
# loopback. Probe that cheaply (no Go, no fixture build) before the heavy
|
||||
# matrix so a misconfigured runner fails fast instead of after 60m.
|
||||
cid="$(docker run -d --name rsmon-worker-test-preflight \
|
||||
-p 127.0.0.1::22 reg.rsxx.ru/library/alpine:3 sleep 300)"
|
||||
port="$(docker port "$cid" 22/tcp 2>/dev/null | sed -n 's#.*:##p' | head -n1)"
|
||||
if [ -z "$port" ] || ! (exec 3<>"/dev/tcp/127.0.0.1/${port}") 2>/dev/null; then
|
||||
cat >&2 <<EOF
|
||||
test-ssh CI: cannot reach a port published on the Docker daemon loopback.
|
||||
|
||||
The harness publishes each fixture's SSH port to the daemon's 127.0.0.1 and
|
||||
then dials 127.0.0.1:<published-port> from the test process, so the job and
|
||||
the Docker daemon must share a loopback network namespace.
|
||||
|
||||
Fix (pick one):
|
||||
1. Run this workflow on a host-mode runner (act_runner job executing on the
|
||||
host with the local Docker daemon), OR
|
||||
2. Give the job container host networking with the Docker socket mounted
|
||||
(e.g. container options: --network host plus /var/run/docker.sock), OR
|
||||
3. Run \`make test-ssh\` directly on a machine with a local Docker daemon.
|
||||
|
||||
Diagnostics:
|
||||
docker server: $(docker version --format '{{.Server.Version}}' 2>/dev/null || echo unknown)
|
||||
published port: ${port:-none}
|
||||
EOF
|
||||
exit 1
|
||||
fi
|
||||
exec 3>&- 2>/dev/null || true
|
||||
|
||||
echo "test-ssh CI: loopback port publishing verified; running the Alpine/Ubuntu/Arch matrix"
|
||||
|
||||
make test-ssh
|
||||
Ссылка в новой задаче
Block a user