Сравнить коммиты
15 Коммитов
fd1a010e31
...
master
| Автор | SHA1 | Дата | |
|---|---|---|---|
|
|
b8c7596fc5 | ||
|
|
714dda08e5 | ||
|
|
674a7d82bf | ||
|
|
bd6070ee1f | ||
|
|
4651deb280 | ||
|
|
cb23f123ae | ||
|
|
a1ccd50aaf | ||
|
|
bf9253d6fe | ||
|
|
9b82c9f82f | ||
|
|
dde720eb44 | ||
|
|
ff0d2f088f | ||
|
|
e987f24903 | ||
|
|
6937674449 | ||
|
|
3256dcdc12 | ||
|
|
579a14b403 |
@@ -2,11 +2,17 @@
|
|||||||
RSMON_URL=https://rsmon.ru
|
RSMON_URL=https://rsmon.ru
|
||||||
RSMON_TOKEN=replace-with-worker-token
|
RSMON_TOKEN=replace-with-worker-token
|
||||||
|
|
||||||
|
# Required for Docker Compose. Replace with a published 64-character digest.
|
||||||
|
RSMON_WORKER_IMAGE_DIGEST=replace-with-64-lowercase-hex-digest
|
||||||
|
|
||||||
# Local operator console. Both credentials are required while the web UI is on.
|
# Local operator console. Both credentials are required while the web UI is on.
|
||||||
WORKER_HOST=0.0.0.0
|
WORKER_HOST=0.0.0.0
|
||||||
WORKER_PORT=27401
|
WORKER_PORT=27401
|
||||||
WORKER_BIND_IP=127.0.0.1
|
WORKER_BIND_IP=127.0.0.1
|
||||||
WORKER_URL=
|
# Advertised public origin: absolute http(s) origin (scheme + host, no path).
|
||||||
|
# PUBLIC_URL is canonical. Legacy WORKER_URL is still read during the bounded
|
||||||
|
# migration but must not be used for new installs.
|
||||||
|
PUBLIC_URL=
|
||||||
WORKER_LOGIN=admin
|
WORKER_LOGIN=admin
|
||||||
WORKER_PASSWORD=replace-with-a-long-random-password
|
WORKER_PASSWORD=replace-with-a-long-random-password
|
||||||
|
|
||||||
|
|||||||
94
.github/workflows/test-ssh.yml
поставляемый
Обычный файл
94
.github/workflows/test-ssh.yml
поставляемый
Обычный файл
@@ -0,0 +1,94 @@
|
|||||||
|
name: SSH Source-Install E2E
|
||||||
|
|
||||||
|
# Work package 5 of docs/source-installation.md: run the Alpine/Ubuntu/Arch
|
||||||
|
# OpenSSH source-install E2E matrix (make test-ssh) in CI.
|
||||||
|
#
|
||||||
|
# Trust boundary. This job builds and runs privileged Docker containers from
|
||||||
|
# repository code (the distro fixtures execute the checked-out source), so it
|
||||||
|
# runs ONLY on trusted refs: pushes to the default branch (master) and manual
|
||||||
|
# workflow_dispatch. It deliberately does NOT trigger on pull_request: a PR
|
||||||
|
# can carry untrusted code into a privileged Docker environment, and Gitea's
|
||||||
|
# fork-PR read-only token clamp does not change what the containers can do on
|
||||||
|
# the runner host. The ordinary unit `make test` CI run (ci.yml) stays
|
||||||
|
# Docker-free and still covers pull requests.
|
||||||
|
#
|
||||||
|
# Runner requirement. The job inherits whatever Docker access `docker.yml`
|
||||||
|
# already relies on. The harness dials fixture SSH ports published on the
|
||||||
|
# Docker daemon loopback, so the runner must expose Docker with loopback port
|
||||||
|
# publishing reachable from the job; scripts/ci/test-ssh.sh fails fast when
|
||||||
|
# that is not the case.
|
||||||
|
#
|
||||||
|
# External network: fixture images (alpine from the reg.rsxx.ru mirror,
|
||||||
|
# ubuntu/arch overridable via RSMON_TEST_IMAGE_*), the pinned Go toolchain
|
||||||
|
# from go.dev, the public source repo from rocketgit.ru, and distro package
|
||||||
|
# repos are fetched live. Operators can pin a resolver for flaky CI DNS via
|
||||||
|
# the RSMON_TEST_DOCKER_DNS repository variable (comma-separated nameservers).
|
||||||
|
|
||||||
|
on:
|
||||||
|
push:
|
||||||
|
branches:
|
||||||
|
- master
|
||||||
|
workflow_dispatch:
|
||||||
|
|
||||||
|
# Per-ref scoping: master pushes cancel a superseded in-flight run on the same
|
||||||
|
# ref instead of stacking; a manual dispatch on another branch has its own
|
||||||
|
# group and never cancels the master run.
|
||||||
|
concurrency:
|
||||||
|
group: test-ssh-${{ gitea.ref }}
|
||||||
|
cancel-in-progress: true
|
||||||
|
|
||||||
|
# Least privilege: the job only needs to read the repository (actions/checkout
|
||||||
|
# and the harness's SSH-free operations). contents:read is supported by Gitea
|
||||||
|
# Actions (the GITEA_TOKEN scope for code/releases).
|
||||||
|
permissions:
|
||||||
|
contents: read
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
test-ssh:
|
||||||
|
name: Alpine/Ubuntu/Arch source-install E2E
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
timeout-minutes: 90
|
||||||
|
env:
|
||||||
|
# All empty by default (harness defaults / Docker embedded DNS).
|
||||||
|
# Override in repo/organization variables:
|
||||||
|
# RSMON_TEST_IMAGE_ALPINE / _UBUNTU / _ARCH - pin a mirror or a
|
||||||
|
# specific distro snapshot for the fixture base image;
|
||||||
|
# RSMON_TEST_DOCKER_DNS - comma-separated nameservers applied as
|
||||||
|
# `docker run --dns ...` for flaky CI resolvers.
|
||||||
|
RSMON_TEST_IMAGE_ALPINE: ${{ vars.RSMON_TEST_IMAGE_ALPINE }}
|
||||||
|
RSMON_TEST_IMAGE_UBUNTU: ${{ vars.RSMON_TEST_IMAGE_UBUNTU }}
|
||||||
|
RSMON_TEST_IMAGE_ARCH: ${{ vars.RSMON_TEST_IMAGE_ARCH }}
|
||||||
|
RSMON_TEST_DOCKER_DNS: ${{ vars.RSMON_TEST_DOCKER_DNS }}
|
||||||
|
steps:
|
||||||
|
# Actions are pinned to immutable full commit SHAs (not moving tags).
|
||||||
|
# Verified 2026-08-13 against the GitHub API that the commit the tag
|
||||||
|
# points to is a commit object:
|
||||||
|
# actions/checkout@v4 -> 11d5960a326750d5838078e36cf38b85af677262
|
||||||
|
# actions/setup-go@v5 -> 40f1582b2485089dde7abd97c1529aa768e1baff
|
||||||
|
# Other workflows (ci.yml, docker.yml) still use moving tags; see the
|
||||||
|
# repo-wide convention note in docs/source-installation.md.
|
||||||
|
- name: Check out code
|
||||||
|
uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262
|
||||||
|
|
||||||
|
- name: Set up Go
|
||||||
|
uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff
|
||||||
|
with:
|
||||||
|
go-version: '1.26.x'
|
||||||
|
# Caches the Go module/build cache used to compile the harness test
|
||||||
|
# binary. Safe: only public dependencies, no credentials.
|
||||||
|
cache: true
|
||||||
|
|
||||||
|
- name: Run Alpine/Ubuntu/Arch OpenSSH source-install E2E
|
||||||
|
run: bash scripts/ci/test-ssh.sh
|
||||||
|
|
||||||
|
# Belt-and-suspenders: the harness and test-ssh.sh already tear down
|
||||||
|
# everything they create; this guarantees a killed job leaves no
|
||||||
|
# rsmon-worker-test-* resources behind on the runner. Filters are
|
||||||
|
# anchored to the harness's own prefix and image repository so cleanup
|
||||||
|
# never touches a shared base image or unrelated resources.
|
||||||
|
- name: Clean up leftover harness resources
|
||||||
|
if: always()
|
||||||
|
run: |
|
||||||
|
docker ps -aq --filter "name=^rsmon-worker-test-" 2>/dev/null | xargs -r docker rm -f >/dev/null 2>&1 || true
|
||||||
|
docker network ls -q --filter "name=^rsmon-worker-test-" 2>/dev/null | xargs -r docker network rm >/dev/null 2>&1 || true
|
||||||
|
docker images -q --filter "reference=rsmon-worker-test/*" 2>/dev/null | xargs -r docker image rm -f >/dev/null 2>&1 || true
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
FROM reg.rsxx.ru/library/golang:1-trixie AS builder
|
FROM golang:1-trixie AS builder
|
||||||
|
|
||||||
WORKDIR /src
|
WORKDIR /src
|
||||||
|
|
||||||
@@ -14,9 +14,9 @@ RUN CGO_ENABLED=0 go build -trimpath \
|
|||||||
-ldflags="-s -w -X main.version=${VERSION} -X main.commit=${COMMIT} -X main.buildDate=${BUILD_DATE}" \
|
-ldflags="-s -w -X main.version=${VERSION} -X main.commit=${COMMIT} -X main.buildDate=${BUILD_DATE}" \
|
||||||
-o /out/rsmon-worker ./cmd/rsmon-worker
|
-o /out/rsmon-worker ./cmd/rsmon-worker
|
||||||
|
|
||||||
FROM reg.rsxx.ru/library/debian:13-slim
|
FROM debian:13-slim
|
||||||
|
|
||||||
RUN apt-get update \
|
RUN DEBIAN_FRONTEND=noninteractive apt-get update \
|
||||||
&& apt-get install -y --no-install-recommends ca-certificates chromium tzdata \
|
&& apt-get install -y --no-install-recommends ca-certificates chromium tzdata \
|
||||||
&& rm -rf /var/lib/apt/lists/* \
|
&& rm -rf /var/lib/apt/lists/* \
|
||||||
&& groupadd --gid 10001 rsmon-worker \
|
&& groupadd --gid 10001 rsmon-worker \
|
||||||
|
|||||||
22
Makefile
22
Makefile
@@ -4,17 +4,24 @@ COMMIT ?= $(shell git rev-parse --short=12 HEAD 2>/dev/null || printf unknown)
|
|||||||
BUILD_DATE ?= $(shell date -u +%Y-%m-%dT%H:%M:%SZ)
|
BUILD_DATE ?= $(shell date -u +%Y-%m-%dT%H:%M:%SZ)
|
||||||
LDFLAGS := -s -w -X main.version=$(VERSION) -X main.commit=$(COMMIT) -X main.buildDate=$(BUILD_DATE)
|
LDFLAGS := -s -w -X main.version=$(VERSION) -X main.commit=$(COMMIT) -X main.buildDate=$(BUILD_DATE)
|
||||||
|
|
||||||
.PHONY: build test check image clean
|
.PHONY: build test test-ssh check image clean
|
||||||
|
|
||||||
build:
|
build:
|
||||||
mkdir -p bin
|
mkdir -p bin
|
||||||
CGO_ENABLED=0 go build -trimpath -ldflags="$(LDFLAGS)" -o $(BINARY) ./cmd/rsmon-worker
|
CGO_ENABLED=0 go build -trimpath -ldflags="$(LDFLAGS)" -o $(BINARY) ./cmd/rsmon-worker
|
||||||
|
|
||||||
test:
|
test:
|
||||||
RSMON_ENV=test CWD=$(CURDIR) go test \
|
# RSMON_TEST_DOCKER=0 pins this target to the non-Docker unit run even
|
||||||
|
# if a developer has the opt-in flag exported in their environment;
|
||||||
|
# only the explicit `make test-ssh` target starts containers.
|
||||||
|
RSMON_ENV=test CWD=$(CURDIR) RSMON_TEST_DOCKER=0 go test \
|
||||||
./cmd/rsmon-worker \
|
./cmd/rsmon-worker \
|
||||||
./internal/distworker \
|
./internal/distworker \
|
||||||
|
./internal/installer \
|
||||||
|
./internal/installer/harness \
|
||||||
|
./internal/sshinstall \
|
||||||
./internal/webapp \
|
./internal/webapp \
|
||||||
|
./internal/compose \
|
||||||
./internal/workercluster \
|
./internal/workercluster \
|
||||||
./internal/wire \
|
./internal/wire \
|
||||||
./internal/checkexec \
|
./internal/checkexec \
|
||||||
@@ -31,6 +38,17 @@ test:
|
|||||||
./checks/cwhois \
|
./checks/cwhois \
|
||||||
./checks/llmhttp
|
./checks/llmhttp
|
||||||
|
|
||||||
|
# test-ssh runs the Docker/OpenSSH source-install harness tests against
|
||||||
|
# real Alpine, Ubuntu, and Arch fixtures, including the full work
|
||||||
|
# package-3 source install (Go toolchain download, clone, and build) on
|
||||||
|
# each. Requires a working Docker daemon; the fixtures pull/start
|
||||||
|
# containers and build inside them, so this is opt-in and is never part
|
||||||
|
# of the default `make test` run. The 60m timeout covers three real
|
||||||
|
# toolchain downloads and worker builds plus the harness lifecycle tests.
|
||||||
|
test-ssh:
|
||||||
|
RSMON_ENV=test CWD=$(CURDIR) RSMON_TEST_DOCKER=1 go test \
|
||||||
|
-v -timeout 60m -count=1 ./internal/installer/harness
|
||||||
|
|
||||||
check:
|
check:
|
||||||
go mod tidy
|
go mod tidy
|
||||||
git diff --exit-code -- go.mod go.sum
|
git diff --exit-code -- go.mod go.sum
|
||||||
|
|||||||
154
README.md
154
README.md
@@ -37,6 +37,11 @@ docker compose up -d
|
|||||||
docker compose logs -f worker
|
docker compose logs -f worker
|
||||||
```
|
```
|
||||||
|
|
||||||
|
Compose constructs an immutable image reference from
|
||||||
|
`RSMON_WORKER_IMAGE_DIGEST`. Set that variable in `.env` to the published
|
||||||
|
64-character lowercase digest before running Compose; a mutable tag cannot be
|
||||||
|
selected through this configuration.
|
||||||
|
|
||||||
The operator console is bound to `127.0.0.1:27401` by default. Set
|
The operator console is bound to `127.0.0.1:27401` by default. Set
|
||||||
`WORKER_BIND_IP` only when a firewall or TLS reverse proxy protects the port.
|
`WORKER_BIND_IP` only when a firewall or TLS reverse proxy protects the port.
|
||||||
Persistent web and cluster state is stored in the `worker-data` volume.
|
Persistent web and cluster state is stored in the `worker-data` volume.
|
||||||
@@ -44,21 +49,27 @@ Persistent web and cluster state is stored in the `worker-data` volume.
|
|||||||
## Docker
|
## Docker
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
docker build -t rsmon-worker:local .
|
IMAGE='reg.rsxx.ru/rsmon/rsmon-worker@sha256:<published-64-character-digest>'
|
||||||
|
docker pull "$IMAGE"
|
||||||
docker run --rm \
|
docker run --rm \
|
||||||
--cap-add NET_RAW \
|
--cap-add NET_RAW \
|
||||||
--env-file .env \
|
--env-file .env \
|
||||||
-p 127.0.0.1:27401:27401 \
|
-p 127.0.0.1:27401:27401 \
|
||||||
-v rsmon-worker-data:/var/lib/rsmon-worker \
|
-v rsmon-worker-data:/var/lib/rsmon-worker \
|
||||||
rsmon-worker:local
|
"$IMAGE"
|
||||||
```
|
```
|
||||||
|
|
||||||
Published images use these tags:
|
Published images have tags for discovery:
|
||||||
|
|
||||||
- `sha-<12-character-commit>` for every push;
|
- `sha-<12-character-commit>` for every push;
|
||||||
- `latest` for `master`;
|
- `latest` for `master`;
|
||||||
- the `v*` release ref, with Docker-invalid characters replaced by `-`.
|
- the `v*` release ref, with Docker-invalid characters replaced by `-`.
|
||||||
|
|
||||||
|
Resolve a trusted published tag through the registry, then deploy the resulting
|
||||||
|
`repository@sha256:...` digest. Tags are mutable and are not accepted by the
|
||||||
|
installer or deploy command; Compose requires the resolved digest to be
|
||||||
|
configured explicitly.
|
||||||
|
|
||||||
The Gitea workflow reads `HARBOR_REGISTRY`, `HARBOR_USER`, and
|
The Gitea workflow reads `HARBOR_REGISTRY`, `HARBOR_USER`, and
|
||||||
`HARBOR_PASSWORD`. `HARBOR_REGISTRY` may be a host such as `reg.rsxx.ru` or an
|
`HARBOR_PASSWORD`. `HARBOR_REGISTRY` may be a host such as `reg.rsxx.ru` or an
|
||||||
HTTP(S) URL; the workflow strips the scheme and trailing slash before composing
|
HTTP(S) URL; the workflow strips the scheme and trailing slash before composing
|
||||||
@@ -66,6 +77,13 @@ Docker image references. The Harbor project is appended separately as `rsmon`.
|
|||||||
|
|
||||||
## systemd
|
## systemd
|
||||||
|
|
||||||
|
The `install` subcommand turns a built binary into an enabled systemd service:
|
||||||
|
it writes a hardened unit, a mode-0600 env file, the data directory, then
|
||||||
|
enables and starts the worker. It reads configuration from flags, `--env-file`,
|
||||||
|
a `.env` in the working directory, or the process environment (in that order),
|
||||||
|
and supports several co-located workers per host via `--name`. The full
|
||||||
|
reference is in [`docs/install.md`](docs/install.md).
|
||||||
|
|
||||||
Install host dependencies first. On Debian or Ubuntu:
|
Install host dependencies first. On Debian or Ubuntu:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
@@ -73,16 +91,43 @@ sudo apt-get update
|
|||||||
sudo apt-get install -y ca-certificates chromium libcap2-bin tzdata
|
sudo apt-get install -y ca-certificates chromium libcap2-bin tzdata
|
||||||
```
|
```
|
||||||
|
|
||||||
Build and install:
|
Build and install with a token file so the token does not enter shell history or
|
||||||
|
the process list:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
make build
|
make build
|
||||||
cp packaging/systemd/worker.env.example worker.env
|
printf '%s\n' 'WORKER_TOKEN' > worker-token
|
||||||
# Edit worker.env.
|
chmod 600 worker-token
|
||||||
sudo ./scripts/install-systemd.sh --env ./worker.env
|
sudo ./bin/rsmon-worker install --token-file worker-token
|
||||||
|
rm worker-token
|
||||||
```
|
```
|
||||||
|
|
||||||
To install an already downloaded release binary:
|
`install` copies the running binary to `/usr/local/bin/rsmon-worker`, writes the
|
||||||
|
mode-0600 configuration at `/etc/rsmon-worker/worker.env`, installs a simple
|
||||||
|
root-run systemd unit, and enables and starts it. Use `--url` to override
|
||||||
|
`https://rsmon.ru`, `--binary` to install another binary, or `--no-start` to
|
||||||
|
configure without starting. When using `--env-file`, installation verifies its
|
||||||
|
`RSMON_URL` and `RSMON_TOKEN` before changing the host. Environment files use
|
||||||
|
portable `KEY=VALUE` lines (plus blank lines and `#` comments); quoting,
|
||||||
|
interpolation, whitespace in values, and YAML-style assignments are rejected
|
||||||
|
because systemd and Docker interpret them differently.
|
||||||
|
|
||||||
|
The Docker alternative pulls the prebuilt image and installs a systemd unit
|
||||||
|
that runs it:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
sudo ./bin/rsmon-worker install --docker --token-file worker-token \
|
||||||
|
--image 'reg.rsxx.ru/rsmon/rsmon-worker@sha256:<published-64-character-digest>'
|
||||||
|
```
|
||||||
|
|
||||||
|
Docker installation requires `--image` with an immutable
|
||||||
|
`repository@sha256:<64-lowercase-hex-characters>` reference. This is a breaking
|
||||||
|
change: prior `--docker` invocations that relied on the `latest` default, or
|
||||||
|
passed only a tag, now fail before Docker is invoked or any host file is changed.
|
||||||
|
|
||||||
|
The token can also be passed as `--token` or `--api-key`, but that can expose it
|
||||||
|
through shell history and process inspection. The legacy repository-based
|
||||||
|
installer remains available:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
sudo ./scripts/install-systemd.sh --binary ./rsmon-worker --env ./worker.env
|
sudo ./scripts/install-systemd.sh --binary ./rsmon-worker --env ./worker.env
|
||||||
@@ -96,9 +141,94 @@ journalctl -u rsmon-worker -f
|
|||||||
sudo systemctl restart rsmon-worker
|
sudo systemctl restart rsmon-worker
|
||||||
```
|
```
|
||||||
|
|
||||||
The service runs as the dedicated `rsmon-worker` user, stores state below
|
The service runs as root, reads secrets from
|
||||||
`/var/lib/rsmon-worker`, reads secrets from `/etc/rsmon-worker/worker.env`, and
|
`/etc/rsmon-worker/worker.env`, and can execute ICMP checks without additional
|
||||||
has only `CAP_NET_RAW` for ICMP checks.
|
capability setup.
|
||||||
|
|
||||||
|
## SSH deployment
|
||||||
|
|
||||||
|
`deploy` uploads the selected worker binary and a temporary mode-0600
|
||||||
|
configuration over SSH, then runs the binary's `install` command through root or
|
||||||
|
`sudo`. The remote host needs Linux, systemd, `base64`, and either root SSH or
|
||||||
|
sudo access.
|
||||||
|
|
||||||
|
```bash
|
||||||
|
./bin/rsmon-worker deploy \
|
||||||
|
--host worker.example.com \
|
||||||
|
--user deploy \
|
||||||
|
--identity-file ~/.ssh/id_ed25519 \
|
||||||
|
--token-file worker-token
|
||||||
|
```
|
||||||
|
|
||||||
|
Add `--docker` to install remotely by uploading only the configuration and
|
||||||
|
systemd unit, then running `docker pull` on the target. This mode does not upload
|
||||||
|
or execute the local worker binary, so the local and remote architectures may
|
||||||
|
differ. It also requires `--image repository@sha256:...`; tag-only references
|
||||||
|
are rejected before connecting to the remote host.
|
||||||
|
|
||||||
|
The default SSH port is 22 and the default RSMon URL is `https://rsmon.ru`.
|
||||||
|
Encrypted keys use `--key-passphrase-file`; password authentication uses
|
||||||
|
`--password-file`; password-protected sudo uses `--sudo-password-file`. Strongly
|
||||||
|
prefer the file options for automation: a secret supplied through a direct flag
|
||||||
|
is visible in the process list and shell history, while a file option never
|
||||||
|
exposes it through argv. The direct flags remain available for interactive
|
||||||
|
convenience.
|
||||||
|
|
||||||
|
SSH host keys are checked against `~/.ssh/known_hosts` by default. Use
|
||||||
|
`--known-hosts PATH` or pin `--host-key-fingerprint SHA256:...`. The explicit
|
||||||
|
`--insecure-host-key` option disables host authentication and should only be
|
||||||
|
used in a trusted disposable environment.
|
||||||
|
|
||||||
|
A Go SSH source installer is implemented as the `source-install` subcommand: it
|
||||||
|
detects the remote distro and architecture over the existing SSH transport,
|
||||||
|
installs the minimal build prerequisites, downloads and SHA-256-verifies the
|
||||||
|
pinned Go 1.26 toolchain, clones/updates the public repository, records the
|
||||||
|
resolved branch and commit, builds the worker to a staging path, and then
|
||||||
|
atomically activates the staged binary together with the validated environment,
|
||||||
|
data directory, and the detected init's service definition. It starts the
|
||||||
|
worker and verifies the process and `/healthz`; any activation, start, or
|
||||||
|
health failure rolls back to the previous working install (preserving the
|
||||||
|
prior binary/env/unit and its enable state), concurrent activations are
|
||||||
|
serialized by a lock, a run killed mid-flight is recovered from its
|
||||||
|
leftover backup marker on the next activation, and reruns are idempotent
|
||||||
|
(exactly one worker process, no leaked temp files). Pass `--no-activate`
|
||||||
|
to stop at the staging build, and `--no-start` to install the layout
|
||||||
|
without starting the worker. The pure detection/planning layer and the
|
||||||
|
Docker/OpenSSH test harness that accepts it are implemented; see
|
||||||
|
[`docs/source-installation.md`](docs/source-installation.md); run the live
|
||||||
|
fixture matrix with `make test-ssh`.
|
||||||
|
|
||||||
|
```bash
|
||||||
|
./bin/rsmon-worker source-install \
|
||||||
|
--host worker.example.com \
|
||||||
|
--user deploy \
|
||||||
|
--identity-file ~/.ssh/id_ed25519 \
|
||||||
|
--token-file ./worker.token
|
||||||
|
```
|
||||||
|
|
||||||
|
By default the installer builds the remote's default branch and records what it
|
||||||
|
resolves to (the public repository currently publishes `master`). Pass
|
||||||
|
`--branch <name>` to pin an explicit branch; it must exist on the remote or the
|
||||||
|
install fails before building. The repository must be an `https://` URL without
|
||||||
|
userinfo.
|
||||||
|
|
||||||
|
The built binary is staged at `/opt/rsmon-worker-src/rsmon-worker` (override with
|
||||||
|
`--build-dir` / `--stage-binary`), the toolchain at `/usr/local/go` (replaced
|
||||||
|
atomically: download, verify, stage, swap with rollback), and the resolved
|
||||||
|
branch and commit in `/opt/rsmon-worker-src/rsmon-worker.commit` (written only
|
||||||
|
after a successful build). Activation installs the binary to
|
||||||
|
`/usr/local/bin/rsmon-worker`, the environment to
|
||||||
|
`/etc/rsmon-worker/worker.env` (mode 0600), the data directory to
|
||||||
|
`/var/lib/rsmon-worker`, and a systemd unit or OpenRC init script for the
|
||||||
|
detected init (an embedded supervisor manages the process when no init is
|
||||||
|
running, as in containers). The env and unit are uploaded into a server-side
|
||||||
|
0700 `mktemp -d` directory so no local user can race a predictable `/tmp` path,
|
||||||
|
and the env file is read/rendered exactly once. The same SSH auth,
|
||||||
|
secret-file, and host-key options as `deploy` apply; the worker token is
|
||||||
|
supplied with `--token`/`--token-file` and written only to the mode-0600 env
|
||||||
|
file. Prefer `--key-passphrase-file`, `--password-file`, and
|
||||||
|
`--sudo-password-file` over their direct-flag equivalents: file options keep
|
||||||
|
secrets out of the process list and shell history.
|
||||||
|
|
||||||
## Configuration
|
## Configuration
|
||||||
|
|
||||||
@@ -108,7 +238,7 @@ has only `CAP_NET_RAW` for ICMP checks.
|
|||||||
| `RSMON_TOKEN` | yes | none | Worker bearer token. |
|
| `RSMON_TOKEN` | yes | none | Worker bearer token. |
|
||||||
| `WORKER_HOST` | no | `0.0.0.0` | Operator-console bind address. |
|
| `WORKER_HOST` | no | `0.0.0.0` | Operator-console bind address. |
|
||||||
| `WORKER_PORT` | no | `27401` | Operator-console port. |
|
| `WORKER_PORT` | no | `27401` | Operator-console port. |
|
||||||
| `WORKER_URL` | no | none | Public URL advertised to the control plane. |
|
| `PUBLIC_URL` | no | none | Advertised public origin (scheme + host, no path). Canonical name; the legacy `WORKER_URL` is still read during the bounded migration in `docs/public-endpoint-and-identity.md`. |
|
||||||
| `WORKER_LOGIN` | yes | none | Operator-console basic-auth login. |
|
| `WORKER_LOGIN` | yes | none | Operator-console basic-auth login. |
|
||||||
| `WORKER_PASSWORD` | yes | none | Operator-console basic-auth password. |
|
| `WORKER_PASSWORD` | yes | none | Operator-console basic-auth password. |
|
||||||
| `RSMON_WEBAPP_DATA_DIR` | no | user data directory | SQLite and local UI state. |
|
| `RSMON_WEBAPP_DATA_DIR` | no | user data directory | SQLite and local UI state. |
|
||||||
|
|||||||
@@ -161,6 +161,85 @@ func TestApplyRemoteCheckResult_DirectWhenQuorumOne(t *testing.T) {
|
|||||||
"the region result row is always inserted even on the legacy path")
|
"the region result row is always inserted even on the legacy path")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestApplyRemoteCheckResultFromWorkerDeduplicatesFailureReplay(t *testing.T) {
|
||||||
|
models.Drop()
|
||||||
|
models.Migrate()
|
||||||
|
|
||||||
|
mon, check := seedAggregatorWorld(t, 1, 5)
|
||||||
|
seedRegion(t, "ru-msk")
|
||||||
|
worker := models.WorkerNode{
|
||||||
|
WorkerID: "replay-worker",
|
||||||
|
RegionCode: "ru-msk",
|
||||||
|
AuthToken: "replay-token",
|
||||||
|
}
|
||||||
|
require.NoError(t, models.DB().Create(&worker).Error)
|
||||||
|
report := makeReport(check.ID, mon.ID, "ERR")
|
||||||
|
report.JobID = "same-failed-execution"
|
||||||
|
|
||||||
|
require.NoError(t, models.ApplyRemoteCheckResultFromWorker(report, worker.RegionCode, &worker))
|
||||||
|
require.NoError(t, models.ApplyRemoteCheckResultFromWorker(report, worker.RegionCode, &worker))
|
||||||
|
|
||||||
|
got := loadCheck(t, check.ID)
|
||||||
|
assert.Equal(t, 1, got.Fails)
|
||||||
|
assert.EqualValues(t, 1, countPendingResults(t, check.ID))
|
||||||
|
var attempts int64
|
||||||
|
require.NoError(t, models.DB().Model(&models.CheckAttempt{}).Where("job_id = ?", report.JobID).Count(&attempts).Error)
|
||||||
|
assert.EqualValues(t, 1, attempts)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestApplyRemoteCheckResultFromWorkerDeduplicatesQuorumReplay(t *testing.T) {
|
||||||
|
models.Drop()
|
||||||
|
models.Migrate()
|
||||||
|
|
||||||
|
mon, check := seedAggregatorWorld(t, 2, 5)
|
||||||
|
seedRegion(t, "ru-msk")
|
||||||
|
worker := models.WorkerNode{
|
||||||
|
WorkerID: "quorum-replay-worker",
|
||||||
|
RegionCode: "ru-msk",
|
||||||
|
AuthToken: "quorum-replay-token",
|
||||||
|
}
|
||||||
|
require.NoError(t, models.DB().Create(&worker).Error)
|
||||||
|
report := makeReport(check.ID, mon.ID, "OK")
|
||||||
|
report.JobID = "same-quorum-execution"
|
||||||
|
|
||||||
|
require.NoError(t, models.ApplyRemoteCheckResultFromWorker(report, worker.RegionCode, &worker))
|
||||||
|
require.NoError(t, models.ApplyRemoteCheckResultFromWorker(report, worker.RegionCode, &worker))
|
||||||
|
|
||||||
|
got := loadCheck(t, check.ID)
|
||||||
|
assert.Equal(t, "UNK", got.State, "quorum state remains owned by the aggregator")
|
||||||
|
assert.EqualValues(t, 1, countPendingResults(t, check.ID))
|
||||||
|
var attempts int64
|
||||||
|
require.NoError(t, models.DB().Model(&models.CheckAttempt{}).Where("job_id = ?", report.JobID).Count(&attempts).Error)
|
||||||
|
assert.EqualValues(t, 1, attempts)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestApplyRemoteCheckResultFromWorkerDeduplicatesFailReplay(t *testing.T) {
|
||||||
|
models.Drop()
|
||||||
|
models.Migrate()
|
||||||
|
|
||||||
|
mon, check := seedAggregatorWorld(t, 1, 5)
|
||||||
|
seedRegion(t, "ru-msk")
|
||||||
|
worker := models.WorkerNode{
|
||||||
|
WorkerID: "fail-replay-worker",
|
||||||
|
RegionCode: "ru-msk",
|
||||||
|
AuthToken: "fail-replay-token",
|
||||||
|
}
|
||||||
|
require.NoError(t, models.DB().Create(&worker).Error)
|
||||||
|
report := makeReport(check.ID, mon.ID, "FAIL")
|
||||||
|
report.JobID = "same-fail-execution"
|
||||||
|
|
||||||
|
require.NoError(t, models.ApplyRemoteCheckResultFromWorker(report, worker.RegionCode, &worker))
|
||||||
|
require.NoError(t, models.ApplyRemoteCheckResultFromWorker(report, worker.RegionCode, &worker))
|
||||||
|
|
||||||
|
got := loadCheck(t, check.ID)
|
||||||
|
assert.Equal(t, "FAIL", got.State)
|
||||||
|
assert.Equal(t, 0, got.Fails, "FAIL resets consecutive ERR tracking")
|
||||||
|
assert.EqualValues(t, 1, countPendingResults(t, check.ID))
|
||||||
|
var attempts int64
|
||||||
|
require.NoError(t, models.DB().Model(&models.CheckAttempt{}).Where("job_id = ?", report.JobID).Count(&attempts).Error)
|
||||||
|
assert.EqualValues(t, 1, attempts)
|
||||||
|
}
|
||||||
|
|
||||||
// TestApplyRemoteCheckResult_BuffersWhenQuorumN pins the new path:
|
// TestApplyRemoteCheckResult_BuffersWhenQuorumN pins the new path:
|
||||||
// when RequireQuorum > 1, ApplyRemoteCheckResult does NOT touch
|
// when RequireQuorum > 1, ApplyRemoteCheckResult does NOT touch
|
||||||
// Check.State — it only inserts the CheckRegionResult row. The check
|
// Check.State — it only inserts the CheckRegionResult row. The check
|
||||||
|
|||||||
@@ -275,91 +275,141 @@ func ApplyRemoteCheckResultFromWorkerTx(tx *gorm.DB, report wire.CheckResultRepo
|
|||||||
return nil, fmt.Errorf("apply check result: nil transaction")
|
return nil, fmt.Errorf("apply check result: nil transaction")
|
||||||
}
|
}
|
||||||
now := time.Now()
|
now := time.Now()
|
||||||
if worker != nil {
|
|
||||||
handled := false
|
handled, err := handleDiagnosticResult(tx, report, worker, now)
|
||||||
if err := ApplyDiagnosticResultTx(tx, report, worker, now, &handled); err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
if handled {
|
if handled {
|
||||||
return nil, nil
|
return nil, nil
|
||||||
}
|
}
|
||||||
}
|
|
||||||
check := Check{}
|
check, err := loadCheckWithMonitor(tx, report.CheckID)
|
||||||
if err := tx.Preload("Monitor").First(&check, report.CheckID).Error; err != nil {
|
if err != nil {
|
||||||
log.Println("worker: check not found:", report.CheckID, err)
|
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
// Always persist the per-region result first so the aggregator can
|
skipped, err := recordCheckAttempt(tx, check, report, worker, now)
|
||||||
// pick it up regardless of which path we take next. We rely on
|
if err != nil {
|
||||||
// StoreCheckRegionResult to default AggregatedAt=NULL (the column
|
return nil, err
|
||||||
// type is *time.Time, so a zero value writes SQL NULL).
|
}
|
||||||
|
if skipped {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
|
||||||
if err := StoreCheckRegionResultTx(tx, report, regionCode, now); err != nil {
|
if err := StoreCheckRegionResultTx(tx, report, regionCode, now); err != nil {
|
||||||
log.Println("worker: error storing region result:", report.CheckID, err)
|
log.Println("worker: error storing region result:", report.CheckID, err)
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
// Quorum-enabled checks: write nothing to Check.State here. The
|
|
||||||
// aggregator will compute the aggregate state once the window has
|
|
||||||
// elapsed (or enough regions have reported) and stamp AggregatedAt on
|
|
||||||
// the contributing CheckRegionResult rows.
|
|
||||||
if check.QuorumEnabled() {
|
if check.QuorumEnabled() {
|
||||||
return nil, nil
|
return nil, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if err := updateCheckState(tx, &check, report, now); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := handleCheckStateSideEffects(tx, &check, report.State, worker, now); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return check.Monitor, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func handleDiagnosticResult(tx *gorm.DB, report wire.CheckResultReport, worker *WorkerNode, now time.Time) (bool, error) {
|
||||||
|
if worker == nil {
|
||||||
|
return false, nil
|
||||||
|
}
|
||||||
|
handled := false
|
||||||
|
if err := ApplyDiagnosticResultTx(tx, report, worker, now, &handled); err != nil {
|
||||||
|
return false, err
|
||||||
|
}
|
||||||
|
return handled, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func loadCheckWithMonitor(tx *gorm.DB, checkID int64) (Check, error) {
|
||||||
|
check := Check{}
|
||||||
|
if err := tx.Preload("Monitor").First(&check, checkID).Error; err != nil {
|
||||||
|
log.Println("worker: check not found:", checkID, err)
|
||||||
|
return check, err
|
||||||
|
}
|
||||||
|
return check, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func recordCheckAttempt(tx *gorm.DB, check Check, report wire.CheckResultReport, worker *WorkerNode, now time.Time) (bool, error) {
|
||||||
|
if worker == nil {
|
||||||
|
return false, nil
|
||||||
|
}
|
||||||
|
payload, _ := json.Marshal(report)
|
||||||
|
attempt := CheckAttempt{JobID: report.JobID, CheckID: check.ID, MonitorID: check.MonitorID, WorkerNodeID: &worker.ID, Kind: AttemptKindRegular, State: AttemptStateFinished, ResultState: report.State, Result: payload, StartedAt: &now, FinishedAt: &now, Deweighted: worker.NetworkProblemActive(now)}
|
||||||
|
if attempt.JobID == "" {
|
||||||
|
attempt.JobID = uuid.New().String()
|
||||||
|
}
|
||||||
|
created := tx.Clauses(clause.OnConflict{
|
||||||
|
Columns: []clause.Column{{Name: "job_id"}},
|
||||||
|
DoNothing: true,
|
||||||
|
}).Create(&attempt)
|
||||||
|
if created.Error != nil {
|
||||||
|
return false, created.Error
|
||||||
|
}
|
||||||
|
return created.RowsAffected == 0, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func updateCheckState(tx *gorm.DB, check *Check, report wire.CheckResultReport, now time.Time) error {
|
||||||
|
update := buildCheckStateUpdate(report, now)
|
||||||
|
if err := tx.Model(check).UpdateColumns(update).Error; err != nil {
|
||||||
|
log.Println("worker: error updating check:", report.CheckID, err)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func buildCheckStateUpdate(report wire.CheckResultReport, now time.Time) map[string]interface{} {
|
||||||
update := map[string]interface{}{
|
update := map[string]interface{}{
|
||||||
colState: report.State,
|
colState: report.State,
|
||||||
colLastEnd: now,
|
colLastEnd: now,
|
||||||
colWarnings: pq.StringArray(report.Warnings),
|
colWarnings: pq.StringArray(report.Warnings),
|
||||||
colInfos: pq.StringArray(report.Infos),
|
colInfos: pq.StringArray(report.Infos),
|
||||||
}
|
}
|
||||||
|
switch report.State {
|
||||||
if report.State == "OK" {
|
case stateOK:
|
||||||
update["was_up"] = now
|
update["was_up"] = now
|
||||||
update["last_ok"] = now
|
update["last_ok"] = now
|
||||||
update["fails"] = 0
|
update["fails"] = 0
|
||||||
update["error"] = gorm.Expr("NULL")
|
update["error"] = gorm.Expr("NULL")
|
||||||
} else {
|
case stateERR:
|
||||||
update["last_fail"] = now
|
update["last_fail"] = now
|
||||||
update["fails"] = gorm.Expr("fails + 1")
|
update["fails"] = gorm.Expr("fails + 1")
|
||||||
if report.Error != nil {
|
if report.Error != nil {
|
||||||
update["error"] = *report.Error
|
update["error"] = *report.Error
|
||||||
}
|
}
|
||||||
|
default:
|
||||||
|
update["last_fail"] = now
|
||||||
|
update["fails"] = 0
|
||||||
|
if report.Error != nil {
|
||||||
|
update["error"] = *report.Error
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if report.ExpiresAt != nil {
|
if report.ExpiresAt != nil {
|
||||||
t, err := time.Parse(time.RFC3339, *report.ExpiresAt)
|
t, err := time.Parse(time.RFC3339, *report.ExpiresAt)
|
||||||
if err == nil {
|
if err == nil {
|
||||||
update["expires"] = t
|
update["expires"] = t
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
return update
|
||||||
|
}
|
||||||
|
|
||||||
if err := tx.Model(&check).UpdateColumns(update).Error; err != nil {
|
func handleCheckStateSideEffects(tx *gorm.DB, check *Check, state string, worker *WorkerNode, now time.Time) error {
|
||||||
log.Println("worker: error updating check:", report.CheckID, err)
|
if worker == nil {
|
||||||
return nil, err
|
return nil
|
||||||
}
|
}
|
||||||
if worker != nil {
|
switch state {
|
||||||
payload, _ := json.Marshal(report)
|
|
||||||
attempt := CheckAttempt{JobID: report.JobID, CheckID: check.ID, MonitorID: check.MonitorID, WorkerNodeID: &worker.ID, Kind: AttemptKindRegular, State: AttemptStateFinished, ResultState: report.State, Result: payload, StartedAt: &now, FinishedAt: &now, Deweighted: worker.NetworkProblemActive(now)}
|
|
||||||
if attempt.JobID == "" {
|
|
||||||
attempt.JobID = uuid.New().String()
|
|
||||||
}
|
|
||||||
// A duplicate websocket/HTTP delivery must not create another attempt.
|
|
||||||
if err := tx.Where("job_id = ?", attempt.JobID).FirstOrCreate(&attempt).Error; err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
switch report.State {
|
|
||||||
case stateERR, stateFail:
|
case stateERR, stateFail:
|
||||||
if err := StartConfirmationTx(tx, check.ID, worker.ID, now); err != nil {
|
return StartConfirmationTx(tx, check.ID, worker.ID, now)
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
case stateOK:
|
case stateOK:
|
||||||
if err := RecoverDiagnosticTx(tx, check.ID, now); err != nil {
|
return RecoverDiagnosticTx(tx, check.ID, now)
|
||||||
return nil, err
|
|
||||||
}
|
}
|
||||||
}
|
return nil
|
||||||
}
|
|
||||||
return check.Monitor, nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// StoreRemoteCheckMetrics persists TSDB points reported by a distributed worker.
|
// StoreRemoteCheckMetrics persists TSDB points reported by a distributed worker.
|
||||||
|
|||||||
@@ -297,7 +297,8 @@ var mutex sync.Mutex
|
|||||||
|
|
||||||
// checkSeverityRank assigns an ordinal to each check state so the monitor
|
// checkSeverityRank assigns an ordinal to each check state so the monitor
|
||||||
// aggregator can pick the highest-severity child deterministically.
|
// aggregator can pick the highest-severity child deterministically.
|
||||||
// Severity order is FAIL > ERR > DEGRADED > WARN > OK — see docs/todo.md
|
// Severity order is FAIL > ERR > DEGRADED > WARN > OK. FAIL means the
|
||||||
|
// worker could not run a check; it remains visible but never opens alerts.
|
||||||
// Phase 3 for the rationale (DEGRADED = partial regional failure, sits
|
// Phase 3 for the rationale (DEGRADED = partial regional failure, sits
|
||||||
// between OK and ERR). Unknown states (UNK, empty, ...) rank 0 so any
|
// between OK and ERR). Unknown states (UNK, empty, ...) rank 0 so any
|
||||||
// real check state takes precedence over them.
|
// real check state takes precedence over them.
|
||||||
@@ -341,7 +342,7 @@ func (m *Monitor) UpdateStatusFromChecks() {
|
|||||||
// inlined three if-statements with non-obvious precedence (a WARN that
|
// inlined three if-statements with non-obvious precedence (a WARN that
|
||||||
// appeared AFTER an ERR in the iteration would never downgrade back,
|
// appeared AFTER an ERR in the iteration would never downgrade back,
|
||||||
// but a FAIL after ERR would silently get clobbered). Using a single
|
// but a FAIL after ERR would silently get clobbered). Using a single
|
||||||
// severity rank keeps the rule FAIL > ERR > DEGRADED > WARN > OK
|
// severity rank keeps the overall health state deterministic.
|
||||||
// independent of slice ordering — the same rule Phase 3 introduces
|
// independent of slice ordering — the same rule Phase 3 introduces
|
||||||
// for DEGRADED, applied uniformly to the existing states too.
|
// for DEGRADED, applied uniformly to the existing states too.
|
||||||
bestRank := checkSeverityRank(stateOK)
|
bestRank := checkSeverityRank(stateOK)
|
||||||
@@ -383,7 +384,7 @@ func (m *Monitor) UpdateStatusFromChecks() {
|
|||||||
// log.Println("state", m.State, "active event:", evt.ID)
|
// log.Println("state", m.State, "active event:", evt.ID)
|
||||||
|
|
||||||
for _, check := range checks { //nolint:gocritic // range copy is acceptable here
|
for _, check := range checks { //nolint:gocritic // range copy is acceptable here
|
||||||
if check.State == stateERR || check.State == stateFail {
|
if check.State == stateERR {
|
||||||
evt.ChecksDown = append(evt.ChecksDown, check.Kind)
|
evt.ChecksDown = append(evt.ChecksDown, check.Kind)
|
||||||
evt.Checks = append(evt.Checks, check)
|
evt.Checks = append(evt.Checks, check)
|
||||||
if check.Error != nil {
|
if check.Error != nil {
|
||||||
@@ -419,7 +420,7 @@ func (m *Monitor) UpdateStatusFromChecks() {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
case stateERR, stateFail:
|
case stateERR:
|
||||||
if evt.ID == 0 {
|
if evt.ID == 0 {
|
||||||
tn := time.Now()
|
tn := time.Now()
|
||||||
evt.StartTime = &tn
|
evt.StartTime = &tn
|
||||||
@@ -500,7 +501,7 @@ func (m *Monitor) syncStatusPageIncidentsTx(tx *gorm.DB, event *Event) error {
|
|||||||
page := &pages[i]
|
page := &pages[i]
|
||||||
var incident StatusPageIncident
|
var incident StatusPageIncident
|
||||||
err := tx.Where("status_page_id = ? AND event_id = ?", page.ID, event.ID).First(&incident).Error
|
err := tx.Where("status_page_id = ? AND event_id = ?", page.ID, event.ID).First(&incident).Error
|
||||||
if m.State == stateERR || m.State == "FAIL" {
|
if m.State == stateERR {
|
||||||
if err != nil {
|
if err != nil {
|
||||||
// The database uniqueness constraint makes concurrent state updates idempotent.
|
// The database uniqueness constraint makes concurrent state updates idempotent.
|
||||||
incident = StatusPageIncident{StatusPageID: page.ID, EventID: &event.ID, Title: m.GetLabel() + " is unavailable", BodyMD: event.Reason, Severity: StatusPageIncidentSeverityCrit, StartedAt: time.Now()}
|
incident = StatusPageIncident{StatusPageID: page.ID, EventID: &event.ID, Title: m.GetLabel() + " is unavailable", BodyMD: event.Reason, Severity: StatusPageIncidentSeverityCrit, StartedAt: time.Now()}
|
||||||
|
|||||||
@@ -251,6 +251,7 @@ func TestTasksForWorker_SkipsLockedAndLeases(t *testing.T) {
|
|||||||
for _, p := range picked {
|
for _, p := range picked {
|
||||||
assert.Equal(t, models.TaskStateLeased, p.State)
|
assert.Equal(t, models.TaskStateLeased, p.State)
|
||||||
assert.Equal(t, w.WorkerID, p.LeaseOwner)
|
assert.Equal(t, w.WorkerID, p.LeaseOwner)
|
||||||
|
assert.NotEmpty(t, p.LeaseToken)
|
||||||
assert.NotNil(t, p.LeaseExpiresAt)
|
assert.NotNil(t, p.LeaseExpiresAt)
|
||||||
assert.Equal(t, 1, p.Attempts)
|
assert.Equal(t, 1, p.Attempts)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -30,7 +30,6 @@ The checker looks for CA certificates in the following locations (in order):
|
|||||||
2. `/etc/ssl/cert.pem` - macOS system certificates
|
2. `/etc/ssl/cert.pem` - macOS system certificates
|
||||||
3. `/etc/pki/tls/certs/ca-bundle.crt` - RHEL/CentOS system certificates
|
3. `/etc/pki/tls/certs/ca-bundle.crt` - RHEL/CentOS system certificates
|
||||||
4. `/usr/local/share/ca-certificates/` - Custom certificate directory
|
4. `/usr/local/share/ca-certificates/` - Custom certificate directory
|
||||||
5. `/data/rsmon/docker/cert-bundles/output/` - Project-specific certificate bundles
|
|
||||||
|
|
||||||
### Mozilla CA Bundle
|
### Mozilla CA Bundle
|
||||||
|
|
||||||
@@ -38,7 +37,7 @@ The project includes the Mozilla CA certificate bundle which contains the same C
|
|||||||
|
|
||||||
```bash
|
```bash
|
||||||
# Downloaded from: https://curl.se/ca/cacert.pem
|
# Downloaded from: https://curl.se/ca/cacert.pem
|
||||||
# Location: docker/cert-bundles/output/mozilla-ca-bundle.crt
|
# Install additional certificates in the operating system trust store.
|
||||||
# Certificate count: 144 CA certificates
|
# Certificate count: 144 CA certificates
|
||||||
```
|
```
|
||||||
|
|
||||||
@@ -125,12 +124,11 @@ To update the CA certificate bundles:
|
|||||||
|
|
||||||
```bash
|
```bash
|
||||||
# Download latest Mozilla CA bundle
|
# Download latest Mozilla CA bundle
|
||||||
cd /data/rsmon
|
curl -fsSL -o /usr/local/share/ca-certificates/mozilla-ca-bundle.crt \
|
||||||
curl -fsSL -o docker/cert-bundles/output/mozilla-ca-bundle.crt \
|
|
||||||
https://curl.se/ca/cacert.pem
|
https://curl.se/ca/cacert.pem
|
||||||
|
|
||||||
# Verify
|
# Verify
|
||||||
grep -c "BEGIN CERTIFICATE" docker/cert-bundles/output/mozilla-ca-bundle.crt
|
grep -c "BEGIN CERTIFICATE" /usr/local/share/ca-certificates/mozilla-ca-bundle.crt
|
||||||
```
|
```
|
||||||
|
|
||||||
## Troubleshooting
|
## Troubleshooting
|
||||||
@@ -141,7 +139,7 @@ This error occurs when no CA certificates can be found. Solutions:
|
|||||||
|
|
||||||
1. Ensure the system has `ca-certificates` package installed
|
1. Ensure the system has `ca-certificates` package installed
|
||||||
2. Place custom CA certificates in `/usr/local/share/ca-certificates/`
|
2. Place custom CA certificates in `/usr/local/share/ca-certificates/`
|
||||||
3. Add certificates to the project bundle at `docker/cert-bundles/output/`
|
3. Add certificates to the operating system trust store
|
||||||
|
|
||||||
### Certificate validation failures
|
### Certificate validation failures
|
||||||
|
|
||||||
|
|||||||
@@ -313,7 +313,6 @@ func LoadBrowserCARoots() (chrome, firefox *x509.CertPool, err error) {
|
|||||||
"/etc/ssl/cert.pem", // macOS
|
"/etc/ssl/cert.pem", // macOS
|
||||||
"/etc/pki/tls/certs/ca-bundle.crt", // RHEL/CentOS
|
"/etc/pki/tls/certs/ca-bundle.crt", // RHEL/CentOS
|
||||||
"/usr/local/share/ca-certificates/", // Custom certs
|
"/usr/local/share/ca-certificates/", // Custom certs
|
||||||
"/data/rsmon/docker/cert-bundles/output/", // Our custom bundled certs
|
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, certPath := range certPaths {
|
for _, certPath := range certPaths {
|
||||||
@@ -389,7 +388,6 @@ func loadFirefoxCARoots(pool *x509.CertPool) bool {
|
|||||||
paths := []string{
|
paths := []string{
|
||||||
"/usr/lib/x86_64-linux-gnu/libnssckbi.so", // Debian/Ubuntu NSS module
|
"/usr/lib/x86_64-linux-gnu/libnssckbi.so", // Debian/Ubuntu NSS module
|
||||||
"/usr/lib/libnssckbi.so", // Generic path
|
"/usr/lib/libnssckbi.so", // Generic path
|
||||||
"/data/rsmon/docker/cert-bundles/output/mozilla/", // Our bundled Mozilla certs
|
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, path := range paths {
|
for _, path := range paths {
|
||||||
|
|||||||
@@ -1,6 +1,9 @@
|
|||||||
package cdns
|
package cdns
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"net"
|
||||||
|
"os"
|
||||||
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/miekg/dns"
|
"github.com/miekg/dns"
|
||||||
@@ -33,7 +36,27 @@ var (
|
|||||||
|
|
||||||
func init() {
|
func init() {
|
||||||
conf = &dns.ClientConfig{
|
conf = &dns.ClientConfig{
|
||||||
Servers: []string{"8.8.8.8", "1.1.1.1", "77.88.8.8"},
|
Servers: resolversFromEnv(),
|
||||||
Port: "53",
|
Port: "53",
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func resolversFromEnv() []string {
|
||||||
|
const defaults = "188.93.17.19,188.93.16.19,82.117.245.122"
|
||||||
|
resolvers := strings.Split(os.Getenv("RSMON_DNS_RESOLVERS"), ",")
|
||||||
|
if strings.TrimSpace(os.Getenv("RSMON_DNS_RESOLVERS")) == "" {
|
||||||
|
resolvers = strings.Split(defaults, ",")
|
||||||
|
}
|
||||||
|
|
||||||
|
valid := make([]string, 0, len(resolvers))
|
||||||
|
for _, resolver := range resolvers {
|
||||||
|
resolver = strings.TrimSpace(resolver)
|
||||||
|
if net.ParseIP(resolver) != nil {
|
||||||
|
valid = append(valid, resolver)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(valid) == 0 {
|
||||||
|
return strings.Split(defaults, ",")
|
||||||
|
}
|
||||||
|
return valid
|
||||||
|
}
|
||||||
|
|||||||
26
checks/cdns/config_test.go
Обычный файл
26
checks/cdns/config_test.go
Обычный файл
@@ -0,0 +1,26 @@
|
|||||||
|
package cdns
|
||||||
|
|
||||||
|
import (
|
||||||
|
"reflect"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestResolversFromEnv(t *testing.T) {
|
||||||
|
t.Setenv("RSMON_DNS_RESOLVERS", " 188.93.17.19, 2001:db8::53, invalid ")
|
||||||
|
|
||||||
|
got := resolversFromEnv()
|
||||||
|
want := []string{"188.93.17.19", "2001:db8::53"}
|
||||||
|
if !reflect.DeepEqual(got, want) {
|
||||||
|
t.Fatalf("resolversFromEnv() = %v, want %v", got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestResolversFromEnvFallsBackToDefaults(t *testing.T) {
|
||||||
|
t.Setenv("RSMON_DNS_RESOLVERS", "not-an-address")
|
||||||
|
|
||||||
|
got := resolversFromEnv()
|
||||||
|
want := []string{"188.93.17.19", "188.93.16.19", "82.117.245.122"}
|
||||||
|
if !reflect.DeepEqual(got, want) {
|
||||||
|
t.Fatalf("resolversFromEnv() = %v, want %v", got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -73,7 +73,9 @@ func Perform(c *models.Check) *Result {
|
|||||||
go localQuery(nsChan, zone, dns.TypeNS)
|
go localQuery(nsChan, zone, dns.TypeNS)
|
||||||
nsResult := <-nsChan
|
nsResult := <-nsChan
|
||||||
if nsResult.r == nil {
|
if nsResult.r == nil {
|
||||||
result.State = stateERR
|
// All recursive resolvers were unreachable, so this worker could not
|
||||||
|
// establish whether the domain is healthy. Do not open an outage event.
|
||||||
|
result.State = "FAIL"
|
||||||
result.Error = fmt.Errorf("cannot retrieve the list of name servers for %s: %s", zone, nsResult.err)
|
result.Error = fmt.Errorf("cannot retrieve the list of name servers for %s: %s", zone, nsResult.err)
|
||||||
return result
|
return result
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -4,7 +4,6 @@ import (
|
|||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"log"
|
"log"
|
||||||
"strings"
|
|
||||||
|
|
||||||
"github.com/miekg/dns"
|
"github.com/miekg/dns"
|
||||||
)
|
)
|
||||||
@@ -14,7 +13,6 @@ func localQuery(mychan chan DNSreply, qname string, qtype uint16) {
|
|||||||
fmt.Printf("DEBUG: start of DNS request \"%s\" / %d\n", qname, qtype)
|
fmt.Printf("DEBUG: start of DNS request \"%s\" / %d\n", qname, qtype)
|
||||||
}
|
}
|
||||||
var result DNSreply
|
var result DNSreply
|
||||||
var trials uint
|
|
||||||
result.qname = qname
|
result.qname = qname
|
||||||
result.qtype = qtype
|
result.qtype = qtype
|
||||||
result.r = nil
|
result.r = nil
|
||||||
@@ -24,41 +22,29 @@ func localQuery(mychan chan DNSreply, qname string, qtype uint16) {
|
|||||||
localm.RecursionDesired = true
|
localm.RecursionDesired = true
|
||||||
localm.Question = make([]dns.Question, 1)
|
localm.Question = make([]dns.Question, 1)
|
||||||
localm.SetEdns0(bufsize, false) // Even if no EDNS requested, see #9 May be we should retry without it if timeout?
|
localm.SetEdns0(bufsize, false) // Even if no EDNS requested, see #9 May be we should retry without it if timeout?
|
||||||
localc := new(dns.Client)
|
|
||||||
localc.ReadTimeout = timeout
|
|
||||||
localm.Question[0] = dns.Question{Name: qname, Qtype: qtype, Qclass: dns.ClassINET}
|
localm.Question[0] = dns.Question{Name: qname, Qtype: qtype, Qclass: dns.ClassINET}
|
||||||
Tests:
|
for _, server := range conf.Servers {
|
||||||
for trials = 0; trials < uint(maxTrials); trials++ {
|
|
||||||
for serverIndex := range conf.Servers {
|
|
||||||
server := conf.Servers[serverIndex]
|
|
||||||
result.nameserver = server
|
result.nameserver = server
|
||||||
// Brackets around the server address are necessary for IPv6 name servers
|
localc := &dns.Client{ReadTimeout: timeout}
|
||||||
// Brackets required for IPv6; do not use net.JoinHostPort (see check-soa commit 3e4edb1)
|
// Brackets are required for IPv6; do not use net.JoinHostPort (see check-soa commit 3e4edb1).
|
||||||
r, rtt, err := localc.Exchange(localm, "["+server+"]:"+conf.Port)
|
r, rtt, err := localc.Exchange(localm, "["+server+"]:"+conf.Port)
|
||||||
if r == nil {
|
if err != nil || r == nil {
|
||||||
result.r = nil
|
result.r = nil
|
||||||
result.err = err
|
result.err = err
|
||||||
log.Println(err.Error())
|
if err != nil {
|
||||||
if strings.Contains(err.Error(), "timeout") {
|
log.Println(err)
|
||||||
// Try another resolver
|
|
||||||
continue
|
|
||||||
}
|
}
|
||||||
// We give in
|
continue
|
||||||
break Tests
|
|
||||||
}
|
}
|
||||||
result.rtt = rtt
|
result.rtt = rtt
|
||||||
if r.Rcode == dns.RcodeSuccess {
|
if r.Rcode == dns.RcodeSuccess {
|
||||||
// TODO: NODATA (NOERROR/ANSWER=0) are silently ignored (e.g. name exists but no IP address)
|
|
||||||
// TODO: for rcodes like SERVFAIL, trying another resolver could make sense
|
|
||||||
result.r = r
|
result.r = r
|
||||||
result.err = nil
|
result.err = nil
|
||||||
break Tests
|
break
|
||||||
}
|
}
|
||||||
// All the other codes are errors
|
|
||||||
result.r = r
|
result.r = r
|
||||||
result.err = errors.New(dns.RcodeToString[r.Rcode])
|
result.err = errors.New(dns.RcodeToString[r.Rcode])
|
||||||
break Tests
|
break
|
||||||
}
|
|
||||||
}
|
}
|
||||||
if debug {
|
if debug {
|
||||||
fmt.Printf("DEBUG: end of DNS request \"%s\" / %d\n", qname, qtype)
|
fmt.Printf("DEBUG: end of DNS request \"%s\" / %d\n", qname, qtype)
|
||||||
|
|||||||
@@ -15,7 +15,6 @@ import (
|
|||||||
"syscall"
|
"syscall"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/hashicorp/raft"
|
|
||||||
"github.com/joho/godotenv"
|
"github.com/joho/godotenv"
|
||||||
|
|
||||||
"rocketgit.ru/rsmon/worker/internal/distworker"
|
"rocketgit.ru/rsmon/worker/internal/distworker"
|
||||||
@@ -33,29 +32,21 @@ var (
|
|||||||
// Phase 1 keeps it on; the flag exists so a Phase 2 basic-auth
|
// Phase 1 keeps it on; the flag exists so a Phase 2 basic-auth
|
||||||
// install can opt out without recompiling.
|
// install can opt out without recompiling.
|
||||||
webappEnabled = true
|
webappEnabled = true
|
||||||
|
|
||||||
// clusterDebugApplyTestConfig, when true, submits the hardcoded
|
|
||||||
// CriticalCheckConfig from workercluster.DefaultDebugCriticalCheck
|
|
||||||
// to the cluster on startup. Wired via the
|
|
||||||
// --cluster-debug-apply-test-config CLI flag; the e2e script
|
|
||||||
// uses this so it can verify FSM replication without the
|
|
||||||
// signed-config-adoption producer (which lands in a later phase).
|
|
||||||
//
|
|
||||||
// DEBUG: this flag is a placeholder. It must be removed (or
|
|
||||||
// guarded behind a build tag) before any production build.
|
|
||||||
//
|
|
||||||
// TODO(phase-N): remove once the real producer is wired.
|
|
||||||
clusterDebugApplyTestConfig = false
|
|
||||||
)
|
)
|
||||||
|
|
||||||
func main() {
|
func main() {
|
||||||
log.SetFlags(log.LstdFlags | log.Lshortfile)
|
log.SetFlags(log.LstdFlags | log.Lshortfile)
|
||||||
|
// Load .env before any subcommand dispatch so management commands
|
||||||
|
// (install/deploy) read the same environment as the worker runtime.
|
||||||
|
// godotenv.Load only fills variables not already present in the
|
||||||
|
// process environment, so an explicit export or sudo -E still wins.
|
||||||
loadDotEnv()
|
loadDotEnv()
|
||||||
|
if handled, code := dispatchManagementCommand(os.Args[1:]); handled {
|
||||||
|
os.Exit(code)
|
||||||
|
}
|
||||||
|
|
||||||
versionFlag := flag.Bool("version", false, "Print version and exit")
|
versionFlag := flag.Bool("version", false, "Print version and exit")
|
||||||
noWeb := flag.Bool("no-web", false, "Disable the local web UI (Phase 1 ships with it on)")
|
noWeb := flag.Bool("no-web", false, "Disable the local web UI (Phase 1 ships with it on)")
|
||||||
debugApplyConfig := flag.Bool("cluster-debug-apply-test-config", false,
|
|
||||||
"Submit a hardcoded CriticalCheckConfig to the cluster on startup. DEBUG: remove once the real config.adopt producer is wired.")
|
|
||||||
flag.Parse()
|
flag.Parse()
|
||||||
|
|
||||||
if *versionFlag {
|
if *versionFlag {
|
||||||
@@ -63,7 +54,6 @@ func main() {
|
|||||||
os.Exit(0)
|
os.Exit(0)
|
||||||
}
|
}
|
||||||
webappEnabled = !*noWeb
|
webappEnabled = !*noWeb
|
||||||
clusterDebugApplyTestConfig = *debugApplyConfig
|
|
||||||
|
|
||||||
if len(flag.Args()) > 0 && flag.Arg(0) == "health" {
|
if len(flag.Args()) > 0 && flag.Arg(0) == "health" {
|
||||||
os.Exit(healthCheck())
|
os.Exit(healthCheck())
|
||||||
@@ -71,6 +61,10 @@ func main() {
|
|||||||
if len(flag.Args()) > 0 && flag.Arg(0) == "liveness" {
|
if len(flag.Args()) > 0 && flag.Arg(0) == "liveness" {
|
||||||
os.Exit(livenessCheck())
|
os.Exit(livenessCheck())
|
||||||
}
|
}
|
||||||
|
if len(flag.Args()) > 0 {
|
||||||
|
fmt.Fprintf(os.Stderr, "unknown command %q\n", flag.Arg(0))
|
||||||
|
os.Exit(2)
|
||||||
|
}
|
||||||
|
|
||||||
log.Println("rsmon-worker starting...")
|
log.Println("rsmon-worker starting...")
|
||||||
|
|
||||||
@@ -82,6 +76,14 @@ func main() {
|
|||||||
if err := webapp.ValidateBasicAuth(cfg.HTTP.Login, cfg.HTTP.Password); err != nil {
|
if err := webapp.ValidateBasicAuth(cfg.HTTP.Login, cfg.HTTP.Password); err != nil {
|
||||||
log.Fatalf("worker: %v", err)
|
log.Fatalf("worker: %v", err)
|
||||||
}
|
}
|
||||||
|
// Enforce the advertised PUBLIC_URL origin rules (and the
|
||||||
|
// production HTTPS policy) before accepting work. willListen=false
|
||||||
|
// because the webapp's both-empty local bcrypt mode is legitimate
|
||||||
|
// and already gated by ValidateBasicAuth above; here we only check
|
||||||
|
// the URL invariants.
|
||||||
|
if err := distworker.ValidateHTTPConfig(cfg.HTTP, false); err != nil {
|
||||||
|
log.Fatalf("worker: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
runner := distworker.NewRunner(&cfg)
|
runner := distworker.NewRunner(&cfg)
|
||||||
|
|
||||||
@@ -288,46 +290,12 @@ func buildCluster(ctx context.Context, cfg *distworker.Config) (*workercluster.C
|
|||||||
log.Printf("worker cluster: started node_id=%s addr=%s bootstrap=%t peers=%d",
|
log.Printf("worker cluster: started node_id=%s addr=%s bootstrap=%t peers=%d",
|
||||||
nodeID, localAddr, bootstrap, len(peers))
|
nodeID, localAddr, bootstrap, len(peers))
|
||||||
|
|
||||||
if clusterDebugApplyTestConfig && bootstrap {
|
|
||||||
// The debug apply only fires on bootstrap nodes; a
|
|
||||||
// joiner cannot commit a log entry until it has been
|
|
||||||
// promoted to voter. Wait for this node to win an
|
|
||||||
// election first (a fresh single-voter cluster elects
|
|
||||||
// itself immediately but the goroutine may run before
|
|
||||||
// the state has flipped).
|
|
||||||
go func() {
|
|
||||||
deadline := time.Now().Add(15 * time.Second)
|
|
||||||
for time.Now().Before(deadline) {
|
|
||||||
if c.Raft() != nil && c.Raft().State() == raft.Leader {
|
|
||||||
break
|
|
||||||
}
|
|
||||||
time.Sleep(100 * time.Millisecond)
|
|
||||||
}
|
|
||||||
if c.Raft() == nil || c.Raft().State() != raft.Leader {
|
|
||||||
log.Printf("worker cluster: applying debug test config id=%d: not leader after 15s",
|
|
||||||
workercluster.DefaultDebugCriticalCheck().ID)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
check := workercluster.DefaultDebugCriticalCheck()
|
|
||||||
applied, err := c.ApplyTestConfig(&check)
|
|
||||||
if err != nil {
|
|
||||||
log.Printf("worker cluster: applying debug test config id=%d: %v",
|
|
||||||
check.ID, err)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
log.Printf("worker cluster: applying debug test config id=%d applied_index=%d",
|
|
||||||
check.ID, applied)
|
|
||||||
}()
|
|
||||||
}
|
|
||||||
|
|
||||||
return c, &clusterAdapter{c: c}, nil
|
return c, &clusterAdapter{c: c}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// clusterAdapter wraps *workercluster.Cluster so it implements the
|
// clusterAdapter wraps *workercluster.Cluster so it implements the
|
||||||
// webapp.ClusterView interface without webapp importing the raft
|
// webapp.ClusterView interface without webapp importing the raft
|
||||||
// code path. The ApplyTestConfig signature is the one webapp expects
|
// code path.
|
||||||
// (no config argument; the cluster subsystem owns the hardcoded
|
|
||||||
// payload so the two sides cannot drift).
|
|
||||||
type clusterAdapter struct {
|
type clusterAdapter struct {
|
||||||
c *workercluster.Cluster
|
c *workercluster.Cluster
|
||||||
}
|
}
|
||||||
@@ -352,11 +320,6 @@ func (a *clusterAdapter) Stats() webapp.ClusterStats {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (a *clusterAdapter) ApplyTestConfig() (uint64, error) {
|
|
||||||
check := workercluster.DefaultDebugCriticalCheck()
|
|
||||||
return a.c.ApplyTestConfig(&check)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (a *clusterAdapter) ClusterID() string { return a.c.ClusterID() }
|
func (a *clusterAdapter) ClusterID() string { return a.c.ClusterID() }
|
||||||
func (a *clusterAdapter) LocalAddr() string { return a.c.LocalAddr() }
|
func (a *clusterAdapter) LocalAddr() string { return a.c.LocalAddr() }
|
||||||
|
|
||||||
@@ -528,6 +491,10 @@ func (w runnerWrapper) RecentNotifications(n int) []webapp.NotificationRow {
|
|||||||
OK: r.OK,
|
OK: r.OK,
|
||||||
Error: r.Error,
|
Error: r.Error,
|
||||||
At: r.At,
|
At: r.At,
|
||||||
|
JobID: r.JobID,
|
||||||
|
Method: r.Method,
|
||||||
|
Status: r.Status,
|
||||||
|
DurationMs: r.DurationMs,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return out
|
return out
|
||||||
@@ -542,20 +509,30 @@ func logHTTPSettings(h distworker.HTTPConfig, willListen bool) {
|
|||||||
if h.Login == "" {
|
if h.Login == "" {
|
||||||
login = "(empty)"
|
login = "(empty)"
|
||||||
}
|
}
|
||||||
url := h.URL
|
url := h.PublicURL
|
||||||
if url == "" {
|
if url == "" {
|
||||||
url = "(empty)"
|
url = "(empty)"
|
||||||
}
|
}
|
||||||
log.Printf("worker http settings: host=%s port=%d url=%s login=%s will_listen=%t",
|
log.Printf("worker http settings: host=%s port=%d public_url=%s login=%s will_listen=%t",
|
||||||
h.Host, h.Port, url, login, willListen)
|
h.Host, h.Port, url, login, willListen)
|
||||||
if h.URL != "" {
|
if h.PublicURL != "" {
|
||||||
if host, warn := distworker.WarnInsecurePublicURL(h.URL); warn {
|
if host, warn := distworker.WarnInsecurePublicURL(h.PublicURL); warn {
|
||||||
|
label := distworker.EnvPublicURL
|
||||||
|
if h.URLSource == distworker.PublicURLSourceLegacy {
|
||||||
|
label = distworker.EnvWorkerURLLegacy
|
||||||
|
}
|
||||||
log.Printf(
|
log.Printf(
|
||||||
"worker http settings: WARN WORKER_URL=http://%s uses plain HTTP on a non-loopback host; "+
|
"worker http settings: WARN %s=http://%s uses plain HTTP on a non-loopback host; "+
|
||||||
"production deployments usually terminate TLS at a reverse proxy", host,
|
"production deployments usually terminate TLS at a reverse proxy", label, host,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if _, source := distworker.PublicURLFromEnv(); source == distworker.PublicURLSourceLegacy {
|
||||||
|
log.Printf(
|
||||||
|
"worker http settings: WARNING WORKER_URL is deprecated (bounded migration); " +
|
||||||
|
"rename it to PUBLIC_URL before it is removed (docs/public-endpoint-and-identity.md milestone 1)",
|
||||||
|
)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func loadDotEnv() {
|
func loadDotEnv() {
|
||||||
|
|||||||
@@ -1,11 +1,45 @@
|
|||||||
package main
|
package main
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"errors"
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/http/httptest"
|
"net/http/httptest"
|
||||||
|
"os"
|
||||||
|
"os/exec"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
func TestDispatchManagementCommand(t *testing.T) {
|
||||||
|
if handled, _ := dispatchManagementCommand(nil); handled {
|
||||||
|
t.Fatal("empty arguments were handled")
|
||||||
|
}
|
||||||
|
if handled, code := dispatchManagementCommand([]string{"install", "--help"}); !handled || code != 0 {
|
||||||
|
t.Fatalf("install help = handled %t code %d", handled, code)
|
||||||
|
}
|
||||||
|
if handled, code := dispatchManagementCommand([]string{"deploy", "--help"}); !handled || code != 0 {
|
||||||
|
t.Fatalf("deploy help = handled %t code %d", handled, code)
|
||||||
|
}
|
||||||
|
if handled, code := dispatchManagementCommand([]string{"source-install", "--help"}); !handled || code != 0 {
|
||||||
|
t.Fatalf("source-install help = handled %t code %d", handled, code)
|
||||||
|
}
|
||||||
|
if handled, code := dispatchManagementCommand([]string{"source-install", "--identity-file", t.TempDir() + "/missing"}); !handled || code == 0 {
|
||||||
|
t.Fatalf("source-install without host/user = handled %t code %d", handled, code)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSecretValue(t *testing.T) {
|
||||||
|
path := t.TempDir() + "/secret"
|
||||||
|
if err := os.WriteFile(path, []byte("value\n"), 0600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
got, err := secretValue("", path)
|
||||||
|
if err != nil || got != "value" {
|
||||||
|
t.Fatalf("secretValue() = %q, %v", got, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestProbeLiveness(t *testing.T) {
|
func TestProbeLiveness(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
@@ -31,3 +65,23 @@ func TestProbeLiveness(t *testing.T) {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestRemovedClusterDebugApplyTestConfigFlagIsRejected(t *testing.T) {
|
||||||
|
binary := filepath.Join(t.TempDir(), "rsmon-worker")
|
||||||
|
build := exec.Command("go", "build", "-o", binary, ".")
|
||||||
|
if output, err := build.CombinedOutput(); err != nil {
|
||||||
|
t.Fatalf("build worker binary: %v\n%s", err, output)
|
||||||
|
}
|
||||||
|
|
||||||
|
output, err := exec.Command(binary, "--cluster-debug-apply-test-config").CombinedOutput()
|
||||||
|
var exitErr *exec.ExitError
|
||||||
|
if !errors.As(err, &exitErr) {
|
||||||
|
t.Fatalf("removed flag error = %v, want parser exit\n%s", err, output)
|
||||||
|
}
|
||||||
|
if exitErr.ExitCode() != 2 {
|
||||||
|
t.Fatalf("removed flag exit code = %d, want 2\n%s", exitErr.ExitCode(), output)
|
||||||
|
}
|
||||||
|
if !strings.Contains(string(output), "flag provided but not defined: -cluster-debug-apply-test-config") {
|
||||||
|
t.Fatalf("removed flag output = %q, want undefined-flag error", output)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
276
cmd/rsmon-worker/management.go
Обычный файл
276
cmd/rsmon-worker/management.go
Обычный файл
@@ -0,0 +1,276 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"flag"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"rocketgit.ru/rsmon/worker/internal/installer"
|
||||||
|
)
|
||||||
|
|
||||||
|
func dispatchManagementCommand(args []string) (bool, int) {
|
||||||
|
if len(args) == 0 {
|
||||||
|
return false, 0
|
||||||
|
}
|
||||||
|
switch args[0] {
|
||||||
|
case "install":
|
||||||
|
return true, installCommand(args[1:])
|
||||||
|
case "deploy":
|
||||||
|
return true, deployCommand(args[1:])
|
||||||
|
case "source-install":
|
||||||
|
return true, sourceInstallCommand(args[1:])
|
||||||
|
default:
|
||||||
|
return false, 0
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func installCommand(args []string) int {
|
||||||
|
fs := flag.NewFlagSet("install", flag.ContinueOnError)
|
||||||
|
fs.SetOutput(os.Stderr)
|
||||||
|
var opts installer.InstallOptions
|
||||||
|
var tokenFile, passwordFile string
|
||||||
|
fs.StringVar(&opts.Binary, "binary", "", "worker binary to install (default: this executable)")
|
||||||
|
fs.StringVar(&opts.EnvFile, "env-file", "", "worker environment file (systemd-safe KEY=VALUE; overrides .env/process env)")
|
||||||
|
fs.StringVar(&opts.Token, "token", "", "worker API token (RSMON_TOKEN)")
|
||||||
|
fs.StringVar(&opts.Token, "api-key", "", "worker API token (alias for --token)")
|
||||||
|
fs.StringVar(&tokenFile, "token-file", "", "file containing the worker API token")
|
||||||
|
fs.StringVar(&opts.URL, "url", "", "RSMon server URL (RSMON_URL; default https://rsmon.ru)")
|
||||||
|
fs.StringVar(&opts.PublicURL, "public-url", "", "advertised public origin (PUBLIC_URL; scheme + host, no path)")
|
||||||
|
fs.StringVar(&opts.Host, "host", "", "operator console bind address (WORKER_HOST; default 127.0.0.1)")
|
||||||
|
fs.StringVar(&opts.Port, "port", "", "operator console port (WORKER_PORT; required with --name)")
|
||||||
|
fs.StringVar(&opts.Login, "login", "", "operator console login (WORKER_LOGIN; default admin with a generated password)")
|
||||||
|
fs.StringVar(&opts.Password, "password", "", "operator console password (WORKER_PASSWORD)")
|
||||||
|
fs.StringVar(&passwordFile, "password-file", "", "file containing the operator console password")
|
||||||
|
fs.StringVar(&opts.Name, "name", "", "instance name: installs a co-located worker (rsmon-worker-<name>) with its own config/data/unit/port")
|
||||||
|
fs.BoolVar(&opts.Docker, "docker", false, "run the prebuilt Docker image instead of the binary")
|
||||||
|
fs.StringVar(&opts.Image, "image", installer.DefaultImage, "immutable Docker repository@sha256 digest required with --docker")
|
||||||
|
fs.BoolVar(&opts.NoStart, "no-start", false, "install and enable without starting")
|
||||||
|
fs.Usage = func() {
|
||||||
|
fmt.Fprintln(fs.Output(), "Usage: rsmon-worker install [--token TOKEN|--env-file FILE] [--name NAME] [--port PORT] [options]")
|
||||||
|
fs.PrintDefaults()
|
||||||
|
}
|
||||||
|
if err := fs.Parse(args); err != nil {
|
||||||
|
if errors.Is(err, flag.ErrHelp) {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
return 2
|
||||||
|
}
|
||||||
|
if fs.NArg() != 0 {
|
||||||
|
fs.Usage()
|
||||||
|
return 2
|
||||||
|
}
|
||||||
|
var err error
|
||||||
|
if opts.Token, err = secretValue(opts.Token, tokenFile); err != nil {
|
||||||
|
fmt.Fprintf(os.Stderr, "install failed: %v\n", err)
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
if opts.Password, err = secretValue(opts.Password, passwordFile); err != nil {
|
||||||
|
fmt.Fprintf(os.Stderr, "install failed: %v\n", err)
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
if err := installer.Install(opts); err != nil {
|
||||||
|
fmt.Fprintf(os.Stderr, "install failed: %v\n", err)
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
func deployCommand(args []string) int {
|
||||||
|
fs := flag.NewFlagSet("deploy", flag.ContinueOnError)
|
||||||
|
fs.SetOutput(os.Stderr)
|
||||||
|
var opts installer.DeployOptions
|
||||||
|
var tokenFile, passwordFile, passphraseFile, sudoPasswordFile string
|
||||||
|
fs.StringVar(&opts.Host, "host", "", "SSH server hostname or address")
|
||||||
|
fs.IntVar(&opts.Port, "port", 22, "SSH server port")
|
||||||
|
fs.StringVar(&opts.User, "user", "", "SSH username")
|
||||||
|
fs.StringVar(&opts.IdentityFile, "identity-file", "", "SSH private key path")
|
||||||
|
fs.StringVar(&opts.KeyPassphrase, "key-passphrase", "", "SSH private key passphrase")
|
||||||
|
fs.StringVar(&passphraseFile, "key-passphrase-file", "", "file containing the private key passphrase")
|
||||||
|
fs.StringVar(&opts.Password, "password", "", "SSH login password")
|
||||||
|
fs.StringVar(&passwordFile, "password-file", "", "file containing the SSH login password")
|
||||||
|
fs.StringVar(&opts.SudoPassword, "sudo-password", "", "remote sudo password")
|
||||||
|
fs.StringVar(&sudoPasswordFile, "sudo-password-file", "", "file containing the remote sudo password")
|
||||||
|
fs.StringVar(&opts.KnownHostsFile, "known-hosts", "", "known_hosts path (default: ~/.ssh/known_hosts)")
|
||||||
|
fs.StringVar(&opts.HostKeyFingerprint, "host-key-fingerprint", "", "expected SHA256 SSH host-key fingerprint")
|
||||||
|
fs.BoolVar(&opts.InsecureHostKey, "insecure-host-key", false, "disable SSH host-key verification (unsafe)")
|
||||||
|
fs.StringVar(&opts.Binary, "binary", "", "worker binary to upload (default: this executable)")
|
||||||
|
fs.StringVar(&opts.Token, "token", "", "worker API token")
|
||||||
|
fs.StringVar(&opts.Token, "api-key", "", "worker API token (alias for --token)")
|
||||||
|
fs.StringVar(&tokenFile, "token-file", "", "file containing the worker API token")
|
||||||
|
fs.StringVar(&opts.URL, "url", installer.DefaultURL, "RSMon server URL")
|
||||||
|
fs.BoolVar(&opts.Docker, "docker", false, "install the prebuilt Docker image remotely")
|
||||||
|
fs.StringVar(&opts.Image, "image", installer.DefaultImage, "immutable Docker repository@sha256 digest required with --docker")
|
||||||
|
fs.BoolVar(&opts.NoStart, "no-start", false, "install and enable without starting")
|
||||||
|
fs.Usage = func() {
|
||||||
|
fmt.Fprintln(fs.Output(), "Usage: rsmon-worker deploy --host HOST --user USER --token TOKEN [SSH options]")
|
||||||
|
fs.PrintDefaults()
|
||||||
|
}
|
||||||
|
if err := fs.Parse(args); err != nil {
|
||||||
|
if errors.Is(err, flag.ErrHelp) {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
return 2
|
||||||
|
}
|
||||||
|
if fs.NArg() != 0 {
|
||||||
|
fs.Usage()
|
||||||
|
return 2
|
||||||
|
}
|
||||||
|
var err error
|
||||||
|
if opts.Token, err = secretValue(opts.Token, tokenFile); err != nil {
|
||||||
|
fmt.Fprintf(os.Stderr, "deploy failed: %v\n", err)
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
if opts.Password, err = secretValue(opts.Password, passwordFile); err != nil {
|
||||||
|
fmt.Fprintf(os.Stderr, "deploy failed: %v\n", err)
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
if opts.KeyPassphrase, err = secretValue(opts.KeyPassphrase, passphraseFile); err != nil {
|
||||||
|
fmt.Fprintf(os.Stderr, "deploy failed: %v\n", err)
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
if opts.SudoPassword, err = secretValue(opts.SudoPassword, sudoPasswordFile); err != nil {
|
||||||
|
fmt.Fprintf(os.Stderr, "deploy failed: %v\n", err)
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
if err := installer.Deploy(opts); err != nil {
|
||||||
|
fmt.Fprintf(os.Stderr, "deploy failed: %v\n", err)
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
fmt.Fprintln(os.Stdout, "rsmon-worker deployed")
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
func secretValue(direct, path string) (string, error) {
|
||||||
|
if direct != "" && path != "" {
|
||||||
|
return "", fmt.Errorf("a secret and its file option cannot both be set")
|
||||||
|
}
|
||||||
|
if path == "" {
|
||||||
|
return direct, nil
|
||||||
|
}
|
||||||
|
b, err := os.ReadFile(path)
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("read secret file: %w", err)
|
||||||
|
}
|
||||||
|
return strings.TrimRight(string(b), "\r\n"), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// sourceInstallCommand drives the remote source installer
|
||||||
|
// (docs/source-installation.md): prerequisites, verified Go toolchain,
|
||||||
|
// clone/update, resolved branch/commit record, a staging build, and (work
|
||||||
|
// package 4) the atomic activation of the staged binary, validated
|
||||||
|
// environment, data dir, and the detected init's service definition,
|
||||||
|
// followed by a start and a process + /healthz verification. It reuses
|
||||||
|
// the deploy SSH options. Any activation/start/health failure rolls back
|
||||||
|
// to the prior working install; --no-activate keeps the staging-only
|
||||||
|
// behavior.
|
||||||
|
func sourceInstallCommand(args []string) int {
|
||||||
|
fs := flag.NewFlagSet("source-install", flag.ContinueOnError)
|
||||||
|
fs.SetOutput(os.Stderr)
|
||||||
|
var opts installer.SourceInstallOptions
|
||||||
|
var passphraseFile, passwordFile, sudoPasswordFile, tokenFile, workerPasswordFile string
|
||||||
|
var noActivate bool
|
||||||
|
fs.StringVar(&opts.Host, "host", "", "SSH server hostname or address")
|
||||||
|
fs.IntVar(&opts.Port, "port", 22, "SSH server port")
|
||||||
|
fs.StringVar(&opts.User, "user", "", "SSH username")
|
||||||
|
fs.StringVar(&opts.IdentityFile, "identity-file", "", "SSH private key path")
|
||||||
|
fs.StringVar(&opts.KeyPassphrase, "key-passphrase", "", "SSH private key passphrase")
|
||||||
|
fs.StringVar(&passphraseFile, "key-passphrase-file", "", "file containing the private key passphrase")
|
||||||
|
fs.StringVar(&opts.Password, "password", "", "SSH login password")
|
||||||
|
fs.StringVar(&passwordFile, "password-file", "", "file containing the SSH login password")
|
||||||
|
fs.StringVar(&opts.SudoPassword, "sudo-password", "", "remote sudo password")
|
||||||
|
fs.StringVar(&sudoPasswordFile, "sudo-password-file", "", "file containing the remote sudo password")
|
||||||
|
fs.StringVar(&opts.KnownHostsFile, "known-hosts", "", "known_hosts path (default: ~/.ssh/known_hosts)")
|
||||||
|
fs.StringVar(&opts.HostKeyFingerprint, "host-key-fingerprint", "", "expected SHA256 SSH host-key fingerprint")
|
||||||
|
fs.BoolVar(&opts.InsecureHostKey, "insecure-host-key", false, "disable SSH host-key verification (unsafe)")
|
||||||
|
fs.StringVar(&opts.Repo, "repo", "", "worker repository to clone/update (default: public rocketgit.ru repo)")
|
||||||
|
fs.StringVar(&opts.Branch, "branch", "", "branch to build; empty resolves the remote default branch")
|
||||||
|
fs.StringVar(&opts.GoVersion, "go-version", "", "Go toolchain version (default: pinned 1.26.0)")
|
||||||
|
fs.StringVar(&opts.GoArch, "go-arch", "", "Go download archive suffix; empty derives it from the remote architecture")
|
||||||
|
fs.StringVar(&opts.BuildDir, "build-dir", "", "remote clone/build directory (default /opt/rsmon-worker-src)")
|
||||||
|
fs.StringVar(&opts.GoModuleProxy, "go-proxy", "", "GOPROXY for the remote build (default: Go default)")
|
||||||
|
fs.StringVar(&opts.ToolchainDir, "toolchain-dir", "", "remote Go install path ending in /go (default /usr/local/go)")
|
||||||
|
fs.StringVar(&opts.StageBinary, "stage-binary", "", "staging binary path (default <build-dir>/rsmon-worker)")
|
||||||
|
fs.DurationVar(&opts.SessionTimeout, "session-timeout", 0, "per-remote-command timeout (default 30m; 0 uses the default)")
|
||||||
|
fs.BoolVar(&noActivate, "no-activate", false, "stop after the staging build and do not install/start a service (activation is the default)")
|
||||||
|
fs.StringVar(&opts.Activation.Name, "name", "", "instance name: activates rsmon-worker-<name> with its own config/data/unit/port")
|
||||||
|
fs.StringVar(&opts.Activation.URL, "url", installer.DefaultURL, "RSMon server URL (RSMON_URL)")
|
||||||
|
fs.StringVar(&opts.Activation.Token, "token", "", "worker API token (RSMON_TOKEN; required for activation)")
|
||||||
|
fs.StringVar(&opts.Activation.Token, "api-key", "", "worker API token (alias for --token)")
|
||||||
|
fs.StringVar(&tokenFile, "token-file", "", "file containing the worker API token")
|
||||||
|
fs.StringVar(&opts.Activation.PublicURL, "public-url", "", "advertised public origin (PUBLIC_URL; scheme + host, no path)")
|
||||||
|
fs.StringVar(&opts.Activation.Host, "worker-host", "", "worker webapp bind address (WORKER_HOST; default 127.0.0.1)")
|
||||||
|
fs.StringVar(&opts.Activation.Port, "worker-port", "", "worker webapp bind port (WORKER_PORT; default 27401, required with --name)")
|
||||||
|
fs.StringVar(&opts.Activation.Login, "worker-login", "", "operator console login (WORKER_LOGIN; default admin with a generated password)")
|
||||||
|
fs.StringVar(&opts.Activation.Password, "worker-password", "", "operator console password (WORKER_PASSWORD)")
|
||||||
|
fs.StringVar(&workerPasswordFile, "worker-password-file", "", "file containing the operator console password")
|
||||||
|
fs.StringVar(&opts.Activation.EnvFile, "env-file", "", "local systemd-safe env file uploaded for activation (overrides the individual knobs)")
|
||||||
|
fs.BoolVar(&opts.Activation.NoStart, "no-start", false, "install the binary, env, data dir, and service definition without starting the worker")
|
||||||
|
fs.Usage = func() {
|
||||||
|
fmt.Fprintln(fs.Output(), "Usage: rsmon-worker source-install --host HOST --user USER [--token TOKEN|--token-file FILE] [SSH options] [source options]")
|
||||||
|
fs.PrintDefaults()
|
||||||
|
}
|
||||||
|
if err := fs.Parse(args); err != nil {
|
||||||
|
if errors.Is(err, flag.ErrHelp) {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
return 2
|
||||||
|
}
|
||||||
|
if fs.NArg() != 0 {
|
||||||
|
fs.Usage()
|
||||||
|
return 2
|
||||||
|
}
|
||||||
|
var err error
|
||||||
|
if opts.KeyPassphrase, err = secretValue(opts.KeyPassphrase, passphraseFile); err != nil {
|
||||||
|
fmt.Fprintf(os.Stderr, "source-install failed: %v\n", err)
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
if opts.Password, err = secretValue(opts.Password, passwordFile); err != nil {
|
||||||
|
fmt.Fprintf(os.Stderr, "source-install failed: %v\n", err)
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
if opts.SudoPassword, err = secretValue(opts.SudoPassword, sudoPasswordFile); err != nil {
|
||||||
|
fmt.Fprintf(os.Stderr, "source-install failed: %v\n", err)
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
if opts.Activation.Token, err = secretValue(opts.Activation.Token, tokenFile); err != nil {
|
||||||
|
fmt.Fprintf(os.Stderr, "source-install failed: %v\n", err)
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
if opts.Activation.Password, err = secretValue(opts.Activation.Password, workerPasswordFile); err != nil {
|
||||||
|
fmt.Fprintf(os.Stderr, "source-install failed: %v\n", err)
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
opts.Activation.Activate = !noActivate
|
||||||
|
res, err := installer.SourceInstall(opts)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Fprintf(os.Stderr, "source-install failed: %v\n", err)
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
fmt.Printf("source install complete\n")
|
||||||
|
fmt.Printf(" distro: %s\n", res.Detection.Summarize())
|
||||||
|
fmt.Printf(" toolchain: %s (%s) at %s\n", res.Plan.Toolchain.Version, res.GoArch, res.ToolchainDir)
|
||||||
|
fmt.Printf(" branch: %s\n", res.ResolvedBranch)
|
||||||
|
fmt.Printf(" commit: %s\n", res.ResolvedCommit)
|
||||||
|
fmt.Printf(" record file: %s\n", res.RecordFile)
|
||||||
|
fmt.Printf(" staged build: %s\n", res.StageBinary)
|
||||||
|
if res.Activation != nil {
|
||||||
|
fmt.Printf(" installed to: %s\n", res.Activation.Binary)
|
||||||
|
fmt.Printf(" env file: %s (mode 0600)\n", res.Activation.EnvFile)
|
||||||
|
fmt.Printf(" data dir: %s\n", res.Activation.DataDir)
|
||||||
|
if res.Activation.UnitFile != "" {
|
||||||
|
fmt.Printf(" service: %s (%s)\n", res.Activation.UnitFile, res.Activation.UnitName)
|
||||||
|
} else {
|
||||||
|
fmt.Println(" service: none (no supported init system detected; supervisor-managed)")
|
||||||
|
}
|
||||||
|
if res.Activation.Started {
|
||||||
|
fmt.Printf(" status: running (supervisor=%s, /healthz verified)\n", res.Activation.Supervisor)
|
||||||
|
} else {
|
||||||
|
fmt.Printf(" status: installed (not started, --no-start)\n")
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
fmt.Println(" status: not installed as a service (--no-activate)")
|
||||||
|
}
|
||||||
|
return 0
|
||||||
|
}
|
||||||
@@ -1,6 +1,6 @@
|
|||||||
services:
|
services:
|
||||||
worker:
|
worker:
|
||||||
image: ${RSMON_WORKER_IMAGE:-reg.rsxx.ru/rsmon/rsmon-worker:latest}
|
image: "reg.rsxx.ru/rsmon/rsmon-worker@sha256:${RSMON_WORKER_IMAGE_DIGEST:?set RSMON_WORKER_IMAGE_DIGEST to the published 64-character digest}"
|
||||||
restart: unless-stopped
|
restart: unless-stopped
|
||||||
env_file:
|
env_file:
|
||||||
- .env
|
- .env
|
||||||
|
|||||||
@@ -27,6 +27,8 @@ disagree, update both in the same change or mark the discrepancy explicitly.
|
|||||||
| [private-workers.md](private-workers.md) | Customer worker isolation, credentials, onboarding | Partial |
|
| [private-workers.md](private-workers.md) | Customer worker isolation, credentials, onboarding | Partial |
|
||||||
| [network-diagnostics.md](network-diagnostics.md) | Confirmation checks and dedicated diagnostic tasks | Partial |
|
| [network-diagnostics.md](network-diagnostics.md) | Confirmation checks and dedicated diagnostic tasks | Partial |
|
||||||
| [critical-check-cluster.md](critical-check-cluster.md) | Raft-backed dispatchless critical checks | Scaffold only |
|
| [critical-check-cluster.md](critical-check-cluster.md) | Raft-backed dispatchless critical checks | Scaffold only |
|
||||||
|
| [public-endpoint-and-identity.md](public-endpoint-and-identity.md) | One HTTPS origin, peer status, CA/mTLS, managed Raft topology | Planned; peer HTTPS partial |
|
||||||
|
| [source-installation.md](source-installation.md) | Go SSH source installer and Docker/OpenSSH test matrix | Partial: harness + planning landed, remote build pending |
|
||||||
| [implementation-roadmap.md](implementation-roadmap.md) | Ordered repository work packages and release gates | Active |
|
| [implementation-roadmap.md](implementation-roadmap.md) | Ordered repository work packages and release gates | Active |
|
||||||
| [source-plan-migration.md](source-plan-migration.md) | Source-to-target conversion ledger and resolved conflicts | Complete mapping |
|
| [source-plan-migration.md](source-plan-migration.md) | Source-to-target conversion ledger and resolved conflicts | Complete mapping |
|
||||||
|
|
||||||
@@ -39,7 +41,7 @@ This repository owns:
|
|||||||
- check and notification execution;
|
- check and notification execution;
|
||||||
- local web console, local state, inventory collection, and host metrics;
|
- local web console, local state, inventory collection, and host metrics;
|
||||||
- worker-to-worker transport and Raft state;
|
- worker-to-worker transport and Raft state;
|
||||||
- Docker, Compose, systemd, and release-image packaging.
|
- Docker, Compose, systemd, source installation, and release-image packaging.
|
||||||
|
|
||||||
The RSMon control-plane repository owns:
|
The RSMon control-plane repository owns:
|
||||||
|
|
||||||
|
|||||||
@@ -78,7 +78,13 @@ provided by a validated control-plane task.
|
|||||||
Raft, and closes SQLite without exceeding the service stop timeout.
|
Raft, and closes SQLite without exceeding the service stop timeout.
|
||||||
- A web-console failure is reported and causes an intentional process policy;
|
- A web-console failure is reported and causes an intentional process policy;
|
||||||
it must not silently leave a partially healthy process.
|
it must not silently leave a partially healthy process.
|
||||||
- Startup rejects malformed `WORKER_URL`, incomplete auth credentials, invalid
|
- Startup validates the advertised origin: canonical `PUBLIC_URL` is held to
|
||||||
|
the strict scheme-and-authority shape (no userinfo, query, fragment, or
|
||||||
|
ambiguous path) and plain HTTP on a non-loopback host is rejected in an
|
||||||
|
explicitly production environment; the legacy `WORKER_URL` is read as a
|
||||||
|
bounded-migration fallback, held only to the tolerant absolute-URL check,
|
||||||
|
and logged with a deprecation warning. Both reject a missing hostname, e.g.
|
||||||
|
`https://:27401`. Startup also rejects incomplete auth credentials, invalid
|
||||||
cluster settings, and unwritable data directories before accepting work.
|
cluster settings, and unwritable data directories before accepting work.
|
||||||
- `GET /healthz` reports process-local liveness. Control-plane reachability is
|
- `GET /healthz` reports process-local liveness. Control-plane reachability is
|
||||||
a separate readiness/selfcheck signal and must not make a healthy container
|
a separate readiness/selfcheck signal and must not make a healthy container
|
||||||
|
|||||||
250
docs/changelog.md
Обычный файл
250
docs/changelog.md
Обычный файл
@@ -0,0 +1,250 @@
|
|||||||
|
# Changelog
|
||||||
|
|
||||||
|
## 2026-08-13
|
||||||
|
|
||||||
|
### Source-install E2E in CI (work package 5)
|
||||||
|
|
||||||
|
- Added `.github/workflows/test-ssh.yml`, a Gitea Actions workflow that runs
|
||||||
|
the Alpine/Ubuntu/Arch Docker/OpenSSH source-install matrix (`make test-ssh`)
|
||||||
|
on pushes to `master` and on manual `workflow_dispatch` only, separate from
|
||||||
|
the Docker-free unit CI. It does not trigger on `pull_request`: the distro
|
||||||
|
fixtures execute the checked-out code inside privileged Docker, so untrusted
|
||||||
|
PR code must never run there automatically. It bounds the job with
|
||||||
|
`timeout-minutes: 90` (the go-test `-timeout 60m` stays in place), scopes
|
||||||
|
concurrency per ref (`test-ssh-${{ gitea.ref }}`), declares
|
||||||
|
`permissions: contents: read`, uploads no artifacts, and cleans up on every
|
||||||
|
path.
|
||||||
|
- Pinned the two actions to immutable full commit SHAs (verified against the
|
||||||
|
GitHub API): `actions/checkout@v4` ->
|
||||||
|
`11d5960a326750d5838078e36cf38b85af677262` and `actions/setup-go@v5` ->
|
||||||
|
`40f1582b2485089dde7abd97c1529aa768e1baff`. The repo-wide convention still
|
||||||
|
leaves `ci.yml`/`docker.yml` on moving tags (accepted, documented risk); see
|
||||||
|
`docs/source-installation.md`.
|
||||||
|
- Wired `RSMON_TEST_IMAGE_ALPINE` / `RSMON_TEST_IMAGE_UBUNTU` /
|
||||||
|
`RSMON_TEST_IMAGE_ARCH` and `RSMON_TEST_DOCKER_DNS` repository variables
|
||||||
|
(all empty by default) so CI can pin per-fixture mirror/snapshot images and
|
||||||
|
a resolver for flaky CI DNS.
|
||||||
|
- Added `scripts/ci/test-ssh.sh`: preflights Docker and the harness's
|
||||||
|
loopback port-publishing requirement with a tiny `docker run -p
|
||||||
|
127.0.0.1::22` probe (fails fast with an actionable message plus
|
||||||
|
diagnostics/fix options instead of a 60m timeout on an unsupported runner),
|
||||||
|
then runs `make test-ssh` and traps `EXIT` to remove every leftover
|
||||||
|
`rsmon-worker-test-*` container/network/image tag. Cleanup filters are
|
||||||
|
anchored to the harness's own prefix/repository so they never touch a shared
|
||||||
|
base image. The workflow adds an `if: always()` cleanup step as a
|
||||||
|
belt-and-suspenders so a killed job never leaves test material on the runner.
|
||||||
|
- External network is fetched live by design (go.dev toolchain, rocketgit.ru
|
||||||
|
source clone, distro repos); operators can pin a resolver via the
|
||||||
|
`RSMON_TEST_DOCKER_DNS` repository variable (comma-separated nameservers,
|
||||||
|
applied as `docker run --dns ...`) and mirror/snapshot overrides via
|
||||||
|
`RSMON_TEST_IMAGE_ALPINE` / `RSMON_TEST_IMAGE_UBUNTU` /
|
||||||
|
`RSMON_TEST_IMAGE_ARCH`.
|
||||||
|
- Fixed a history-dependent test fragility: `TestSourceInstallDirtyCheckoutPreservesStaging`
|
||||||
|
and the rollback test's build-failure step dirty the tracked tree by
|
||||||
|
appending a marker line to `Makefile` instead of `git checkout master~1 --
|
||||||
|
Makefile`, which silently stopped dirtying the tree once the last commit did
|
||||||
|
not touch that file.
|
||||||
|
|
||||||
|
## 2026-08-12
|
||||||
|
|
||||||
|
### Source-install hardening review
|
||||||
|
|
||||||
|
- Fail-closed remote scripts: checkout, branch resolution, and build steps now
|
||||||
|
run under `set -eu` (and package/record steps chain with `&&`), so a failed
|
||||||
|
checkout or fetch can never be masked by a stale `rev-parse` or subsequent
|
||||||
|
command. The checkout step additionally refuses (`git diff --quiet` /
|
||||||
|
`--cached --quiet`) before the destructive `checkout -B`, because `-B`
|
||||||
|
silently discards local changes and would otherwise never fail on a dirty
|
||||||
|
tree. A dirty-tree checkout failure surfaces as a `check out branch` error
|
||||||
|
before the build runs; the new `TestSourceInstallSSHCheckoutFailureNotMasked`
|
||||||
|
unit test and `TestSourceInstallDirtyCheckoutPreservesStaging` Docker test
|
||||||
|
prove the previous staging binary and commit record are preserved
|
||||||
|
byte-for-byte.
|
||||||
|
- Atomic toolchain replacement: the Go toolchain is downloaded, SHA-256
|
||||||
|
verified, extracted into a same-filesystem staging dir, verified to report
|
||||||
|
the target version, and only then swapped into `ToolchainDir` with the prior
|
||||||
|
toolchain moved to a sibling `.go-backup` that is restored on swap failure.
|
||||||
|
A failed download/verify/extract/swap never destroys the prior Go.
|
||||||
|
- Record-after-build pairing: `rsmon-worker.commit` is written only after a
|
||||||
|
successful build, so the record and the staged binary always correspond to
|
||||||
|
the same commit. The build verifies `<stage>.new --version` before an atomic
|
||||||
|
`mv -f` over the previous staging binary; `GOMODCACHE` is now set alongside
|
||||||
|
`GOCACHE` inside the build dir so reruns reuse both caches.
|
||||||
|
- Origin verification: an existing checkout's `remote.origin.url` must exactly
|
||||||
|
match the configured repository before anything is fetched or built.
|
||||||
|
- Repository hardening: only `https://` clone URLs without userinfo are
|
||||||
|
accepted (`ValidateRepoURL`, enforced before dialing and again when
|
||||||
|
planning).
|
||||||
|
- Explicit charset validation for Go version and architecture overrides
|
||||||
|
(`sshinstall.ValidGoVersion` / `ValidGoArch`) before any remote mutation.
|
||||||
|
- Bounded remote execution: each remote command is capped by
|
||||||
|
`--session-timeout` (default 30m) and captured stdout is size-bounded
|
||||||
|
alongside the existing stderr bound; deploy's streaming `runRemote` keeps its
|
||||||
|
historical no-timeout behavior.
|
||||||
|
- The source installer now defaults to the remote's default branch (the public
|
||||||
|
repo publishes `master`) instead of the plan's stale `main` default, while
|
||||||
|
`--branch` still pins an explicit branch that must exist remotely. The
|
||||||
|
README quickstart no longer shows the incorrect `--branch main`.
|
||||||
|
- CLI secret flags keep their compatibility, but docs now explicitly state that
|
||||||
|
file options (`-password-file`, etc.) keep secrets out of argv and shell
|
||||||
|
history while direct flags expose them through the process list.
|
||||||
|
- The harness accepts `RSMON_TEST_DOCKER_DNS` (comma-separated) to pin
|
||||||
|
`docker run --dns` for fixture containers, so internet-facing installs are
|
||||||
|
not at the mercy of a flaky local resolver.
|
||||||
|
|
||||||
|
### Remote source-install execution (work package 3)
|
||||||
|
|
||||||
|
- Added `installer.SourceInstall` (`internal/installer/sourceinstall.go`):
|
||||||
|
executes the source-install flow through the existing SSH transport,
|
||||||
|
reusing the `deploy` command's `SSHOptions` (keys, passphrases,
|
||||||
|
passwords, sudo passwords, known-hosts, pinned fingerprints) and its
|
||||||
|
privilege path. Extracted the shared `SSHOptions` struct and a
|
||||||
|
`sudoWrap` helper so deploy and source install cannot diverge.
|
||||||
|
- Steps implemented: minimal package-prerequisite install per distro
|
||||||
|
(`apk`/`apt`/`pacman`/`dnf`, never a compiler), SHA-256-verified Go 1.26
|
||||||
|
toolchain download/extraction with an idempotent version-skip and temp-dir
|
||||||
|
cleanup, clone-or-update of the public repository (with a bounded 3-attempt
|
||||||
|
retry for transient DNS/TLS/proxy failures), resolution of the remote
|
||||||
|
default branch (a pinned branch must exist remotely), a resolved branch and
|
||||||
|
commit record at `<BuildDir>/rsmon-worker.commit`, and a staging build
|
||||||
|
(`CGO_ENABLED=0`, `-trimpath`, repository `-ldflags`) verified via
|
||||||
|
`--version`. The running service, config, and data directory are untouched
|
||||||
|
(work package 4 boundary).
|
||||||
|
- Security: every interpolated remote value is single-quoted; branch and
|
||||||
|
commit values are strictly validated; no worker token or control-plane
|
||||||
|
credential is sent; sudo passwords travel only over session stdin; remote
|
||||||
|
errors are bounded (stderr truncated in `runRemoteOutput`).
|
||||||
|
- Added unit tests for the remote scripts, option validation, branch/commit
|
||||||
|
parsing, sudo wrapping, the secrets-absent contract, and an in-process
|
||||||
|
real-SSH orchestration flow (with missing-pinned-branch, build-failure, and
|
||||||
|
detection-failure paths).
|
||||||
|
- Added `TestSourceInstallFixtures` to the Docker/OpenSSH harness: each of
|
||||||
|
Alpine, Ubuntu, and Arch installs from a clean state through the real
|
||||||
|
harness transport (prerequisite install, verified Go 1.26, clone, resolved
|
||||||
|
commit, staging build), then a rerun proves idempotency (same branch,
|
||||||
|
toolchain reuse, no temp leaks). All three resolved the public repo's
|
||||||
|
`master` at `4651deb2...` in the recorded run. `make test-ssh` timeout
|
||||||
|
raised to 60m.
|
||||||
|
- Documented the branch-resolution reality: the public repository currently
|
||||||
|
publishes `master`, and the installer records whatever the remote default
|
||||||
|
branch resolves to.
|
||||||
|
|
||||||
|
### Source-install harness and planning foundations (work packages 1-2)
|
||||||
|
|
||||||
|
- Added `internal/installer/harness`: a reusable Docker/OpenSSH test harness
|
||||||
|
that builds real OpenSSH containers for Alpine, Ubuntu, and Arch, waits for
|
||||||
|
real network SSH readiness, captures the server host key into a temp
|
||||||
|
`known_hosts` file, and tears the container, network, per-instance fixture
|
||||||
|
image tag, and temp dir down reliably. It uses the `golang.org/x/crypto/ssh`
|
||||||
|
library and known_hosts verification semantics the installer's `deploy` path
|
||||||
|
relies on (the harness owns its connection code rather than reusing the
|
||||||
|
installer functions) and never mocks SSH. A fresh known_hosts file trusts
|
||||||
|
the first key (TOFU); the host-key mismatch test proves a different key is
|
||||||
|
rejected before any command runs.
|
||||||
|
- Added distro fixtures under `internal/installer/harness/testdata/fixtures`.
|
||||||
|
Alpine defaults to the `reg.rsxx.ru/library/alpine:3` mirror; Ubuntu and Arch
|
||||||
|
fall back to Docker Hub refs overridable via `RSMON_TEST_IMAGE_<NAME>`.
|
||||||
|
Each fixture starts clean (no Go, no worker source) and authenticates with a
|
||||||
|
bundled test key; password auth is disabled. The test key is strictly
|
||||||
|
test-only - it grants root only to the disposable fixture containers - and
|
||||||
|
must never be used outside the harness.
|
||||||
|
- Added opt-in integration controls: the Docker tests run only with
|
||||||
|
`RSMON_TEST_DOCKER=1` (`make test-ssh`); default `make test` and `go test
|
||||||
|
./...` skip them and never pull or start containers. `make test` also pins
|
||||||
|
`RSMON_TEST_DOCKER=0` so an exported opt-in flag cannot leak into the unit
|
||||||
|
run.
|
||||||
|
- Integration tests assert real SSH round trips, clean target state, distro /
|
||||||
|
package-manager / init detection per fixture, a full source plan including
|
||||||
|
the pinned Go toolchain, host-key mismatch rejection, host-key stability,
|
||||||
|
failed-start cleanup, and complete teardown (container, network, fixture
|
||||||
|
image tag, and temp dir gone).
|
||||||
|
- Added `internal/sshinstall`: pure, unit-tested detection and planning for the
|
||||||
|
source installer - os-release parsing, distro/package-manager/init
|
||||||
|
resolution, `uname -m` to Go archive mapping, pinned Go 1.26 toolchain with
|
||||||
|
published SHA-256, and a reviewable ordered plan. No remote execution yet.
|
||||||
|
- `make test` now includes the new packages; `make test-ssh` runs the live
|
||||||
|
fixture matrix.
|
||||||
|
|
||||||
|
### Public endpoint configuration (milestone 1 of public-endpoint-and-identity)
|
||||||
|
|
||||||
|
- `PUBLIC_URL` is now the canonical advertised public origin; the legacy
|
||||||
|
`WORKER_URL` is accepted only for the bounded migration and logs a startup
|
||||||
|
deprecation warning. `PUBLIC_URL` wins whenever both are set, and the
|
||||||
|
installer drops `WORKER_URL` from freshly written env files when
|
||||||
|
`PUBLIC_URL` is present.
|
||||||
|
- Startup and install validate the origin shape: absolute `http`/`https` URL
|
||||||
|
with scheme and authority only; userinfo, query, fragment, and any path
|
||||||
|
other than `/` are rejected.
|
||||||
|
- Plain-HTTP `PUBLIC_URL` on a non-loopback host is rejected in an explicitly
|
||||||
|
production environment (`DEPLOY_ENV`, `RSMON_ENV`, or `GO_ENV` =
|
||||||
|
`production`); other environments keep the historical warning.
|
||||||
|
- `internal/wire` adds `public_url` to `WorkerInit` (control plane to worker),
|
||||||
|
keeping the legacy `url` field for old control planes; the worker prefers
|
||||||
|
`public_url` and rejects unusable values, keeping the previous accepted
|
||||||
|
URL. `RegisterRequest.public_url` is the registration contract for the
|
||||||
|
pending RSMon counterpart (the worker does not currently transmit the URL
|
||||||
|
during registration; it consumes the accepted endpoint from `WorkerInit`).
|
||||||
|
- The legacy `WORKER_URL` is held only to the tolerant absolute-URL check
|
||||||
|
(no newly rejected legacy shapes); `PUBLIC_URL` is held to the strict
|
||||||
|
scheme-and-authority origin shape. Both reject a missing hostname, e.g.
|
||||||
|
`https://:27401`.
|
||||||
|
|
||||||
|
## 2026-07-19
|
||||||
|
|
||||||
|
### Standalone installation and deployment
|
||||||
|
|
||||||
|
- Added `rsmon-worker install` for installing the current binary, a mode-0600
|
||||||
|
environment file, and a root-owned systemd service.
|
||||||
|
- Added `rsmon-worker deploy` for installing workers over SSH with key or
|
||||||
|
password authentication, optional secret files, and host-key verification
|
||||||
|
through `known_hosts` or a pinned fingerprint.
|
||||||
|
- Added `--token-file`, `--url`, `--api-key`, and `--no-start` deployment
|
||||||
|
options.
|
||||||
|
- Added optional `--docker` deployment using a prebuilt image. Docker install
|
||||||
|
and deploy now require an immutable `repository@sha256:...` reference before
|
||||||
|
any Docker or remote-host mutation; the former mutable `latest` default is no
|
||||||
|
longer accepted.
|
||||||
|
- Simplified the default systemd service to `Type=simple`, `User=root`, and
|
||||||
|
`Restart=on-failure`.
|
||||||
|
- Reworked the legacy `scripts/install-systemd.sh` script as a compatibility
|
||||||
|
wrapper around the built-in installer.
|
||||||
|
|
||||||
|
### Standalone repository cleanup
|
||||||
|
|
||||||
|
- Removed the remaining certificate-bundle fallback under `/data/rsmon` and
|
||||||
|
updated its documentation.
|
||||||
|
- Changed Docker build and runtime bases to public Go and Debian images.
|
||||||
|
- Documented binary, systemd, Docker, and SSH deployment workflows.
|
||||||
|
- Added installer and SSH host-verification tests.
|
||||||
|
|
||||||
|
### Verification
|
||||||
|
|
||||||
|
- Passed `make test`, `make build`, `go vet`, and `go mod verify`.
|
||||||
|
- Validated systemd units, Compose configuration, Docker image pull and
|
||||||
|
execution, and production worker job reporting.
|
||||||
|
- Published the changes as commit `3256dcd` (`feat: add worker install and
|
||||||
|
deploy`) on `master`, triggering the Docker image workflow.
|
||||||
|
|
||||||
|
### Task protocol and local audit hardening
|
||||||
|
|
||||||
|
- Required one task-envelope branch, matching outer/inner job IDs, and a
|
||||||
|
non-empty lease token before local execution.
|
||||||
|
- Added structured terminal failures for unsupported check kinds and safely
|
||||||
|
attributable malformed envelopes.
|
||||||
|
- Recorded delegated notification outcomes in the bounded `/notifications`
|
||||||
|
view using only job ID, method, status, duration, and time.
|
||||||
|
- Added static permanent handling for invalid deadlines and recovered
|
||||||
|
notification executor panics without retaining secret-bearing text.
|
||||||
|
- Removed the critical-cluster test-config endpoint, CLI flag, environment
|
||||||
|
switch, and production helper; hardcoded config application is test-only.
|
||||||
|
- Made runner token rotation connection-scoped and in-memory: it reconnects
|
||||||
|
without stopping web, inventory, metrics, or cluster subsystems. Durable token
|
||||||
|
storage, bootstrap exchange, rotation acknowledgement, and revocation remain
|
||||||
|
unimplemented.
|
||||||
|
- Added bounded resend of dequeued check and notification result envelopes after
|
||||||
|
websocket reconnect; control-plane application remains at-least-once and must
|
||||||
|
deduplicate by leased job and lease token. Failed metric snapshots are dropped
|
||||||
|
and replaced by the next periodic tick, not replayed.
|
||||||
|
- SIGTERM stops new dispatch and waits for active work, but stale-lease
|
||||||
|
acknowledgement, bounded graceful final-result drain, and duplicate-frame
|
||||||
|
coverage remain open.
|
||||||
@@ -16,6 +16,9 @@ GET /worker?token=<RSMON_TOKEN>
|
|||||||
`/api/worker` and the HTTP jobs/results APIs remain compatibility paths. New
|
`/api/worker` and the HTTP jobs/results APIs remain compatibility paths. New
|
||||||
workers use WebSocket task envelopes.
|
workers use WebSocket task envelopes.
|
||||||
|
|
||||||
|
Public endpoint and cluster identity requirements are defined in
|
||||||
|
[public-endpoint-and-identity.md](public-endpoint-and-identity.md).
|
||||||
|
|
||||||
## Frame Model
|
## Frame Model
|
||||||
|
|
||||||
Every frame is a `wire.WorkerMessage` with `kind` and one active content
|
Every frame is a `wire.WorkerMessage` with `kind` and one active content
|
||||||
@@ -35,27 +38,47 @@ branch.
|
|||||||
| Server to worker | diagnostic task | new task-envelope variant | Planned |
|
| Server to worker | diagnostic task | new task-envelope variant | Planned |
|
||||||
|
|
||||||
The legacy top-level `task` and `notification_task` branches are accepted for
|
The legacy top-level `task` and `notification_task` branches are accepted for
|
||||||
rollout compatibility. If a frame includes both a valid current
|
rollout compatibility. A current `task_envelope` must activate exactly one
|
||||||
`task_envelope` and a legacy branch, the worker executes only
|
matching branch with a non-empty, matching inner and outer job ID and lease
|
||||||
`task_envelope`. The current runner ignores malformed or unsupported task
|
token. Unsupported check kinds and attributable malformed envelopes produce
|
||||||
branches and does not yet compare the envelope job ID with the inner job ID;
|
terminal reports; malformed envelopes without a usable identity are rejected
|
||||||
strict rejection and reporting are P1 work below.
|
without execution or reporting.
|
||||||
|
|
||||||
## Initialization And Refresh
|
## Initialization And Refresh
|
||||||
|
|
||||||
`wire.WorkerInit` supplies runtime values owned by the control plane:
|
The worker is expected to propose `PUBLIC_URL` during registration and the
|
||||||
|
control plane to validate and canonicalize it; `wire.WorkerInit` returns the
|
||||||
|
accepted endpoint and supplies runtime values owned by the control plane:
|
||||||
|
|
||||||
- worker ID, region, advertised URL, capabilities, and concurrency;
|
- worker ID, region, advertised URL, capabilities, and concurrency;
|
||||||
- allowed notification methods and account IDs;
|
- allowed notification methods and account IDs;
|
||||||
- optional linked server ID for host metrics;
|
- optional linked server ID for host metrics;
|
||||||
- LLM endpoints;
|
- LLM endpoints;
|
||||||
- scoped notification credentials and system contacts;
|
- scoped notification credentials and system contacts;
|
||||||
- peer workers for selfcheck/cluster-adjacent behavior.
|
- signed, cluster-scoped peer topology for selfcheck and Raft behavior.
|
||||||
|
|
||||||
|
Current worker behavior: the worker validates its local `PUBLIC_URL`
|
||||||
|
configuration at startup and *consumes* the accepted endpoint from
|
||||||
|
`wire.WorkerInit` (preferring `public_url`, falling back to the legacy `url`
|
||||||
|
field). Transmitting the proposed URL during registration is the pending RSMon
|
||||||
|
control-plane counterpart; the worker does not currently send it.
|
||||||
|
|
||||||
|
Worker ID, account, region, cluster, membership, role, topology generation, and
|
||||||
|
certificate identity are control-plane authority. Local environment or a peer
|
||||||
|
response cannot override them. Static peer environment remains lab-only.
|
||||||
|
|
||||||
The worker clamps supplied concurrency to its local maximum. Credentials are
|
The worker clamps supplied concurrency to its local maximum. Credentials are
|
||||||
replaced atomically in memory on refresh. Removed credentials must become
|
replaced atomically in memory on refresh. Removed credentials must become
|
||||||
unavailable immediately after the refresh is applied.
|
unavailable immediately after the refresh is applied.
|
||||||
|
|
||||||
|
`wire.WorkerInit` returns the accepted endpoint as `public_url`, with the
|
||||||
|
legacy `url` field still populated during the bounded migration; the worker
|
||||||
|
prefers `public_url` and ignores an unusable value (keeping the previous
|
||||||
|
accepted URL). `RegisterRequest.public_url` is the registration contract the
|
||||||
|
RSMon control-plane counterpart must populate when it wires worker-initiated
|
||||||
|
registration; the worker does not transmit it today. See
|
||||||
|
[public-endpoint-and-identity.md](public-endpoint-and-identity.md).
|
||||||
|
|
||||||
Private-worker hardening will add an immutable worker account ID, config
|
Private-worker hardening will add an immutable worker account ID, config
|
||||||
version, expiry, and signature. Until then the executable trusts the
|
version, expiry, and signature. Until then the executable trusts the
|
||||||
authenticated control plane to send a correctly scoped config; server-side
|
authenticated control plane to send a correctly scoped config; server-side
|
||||||
@@ -86,9 +109,10 @@ selection remains the primary isolation boundary.
|
|||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
`type=notification` activates `notification` instead. Job IDs in the envelope
|
`type=notification` activates `notification` instead. The worker enforces the
|
||||||
and active branch must match once P1 validation lands. Today a task without a
|
matching job IDs and non-empty lease token before dispatch. A task without one
|
||||||
recognized populated branch is ignored without execution.
|
recognized populated branch is rejected without execution; it is reported only
|
||||||
|
when the active branch provides attributable job and lease identity.
|
||||||
|
|
||||||
## Result Invariants
|
## Result Invariants
|
||||||
|
|
||||||
@@ -117,20 +141,26 @@ Files:
|
|||||||
- `internal/wire/types_test.go`
|
- `internal/wire/types_test.go`
|
||||||
- `internal/distworker/runner_protocol_test.go`
|
- `internal/distworker/runner_protocol_test.go`
|
||||||
|
|
||||||
Tests:
|
Implemented:
|
||||||
|
|
||||||
- decode every current frame branch;
|
- strict single-branch envelope selection with matching job-ID and lease
|
||||||
- prefer the current envelope over duplicate legacy branches;
|
validation;
|
||||||
- reject mismatched envelope and inner job IDs;
|
- unsupported-kind and attributable malformed-task terminal reporting;
|
||||||
- preserve unknown optional fields during compatible rollout;
|
- atomic config and credential replacement.
|
||||||
- reject missing lease tokens before execution result submission;
|
|
||||||
- atomically replace config and credentials.
|
Still required:
|
||||||
|
|
||||||
|
- decode conformance coverage for every current frame branch and unknown
|
||||||
|
optional fields;
|
||||||
|
- explicit drain behavior and stale-lease acknowledgement coverage.
|
||||||
|
|
||||||
### P2: Safe Token Rotation
|
### P2: Safe Token Rotation
|
||||||
|
|
||||||
Current `Runner.RotateToken` closes the runner stop channel. Replace this with a
|
`Runner.RotateToken` replaces the token in memory, closes only the active
|
||||||
connection-scoped cancellation path that stores the new token, closes only the
|
control-plane WebSocket, and reconnects without terminating worker services.
|
||||||
active WebSocket, and reconnects without terminating worker services.
|
The replacement is not persisted: a process restart still uses its configured
|
||||||
|
startup token. The control plane must tolerate connection-scoped result resend
|
||||||
|
using the leased job and lease token for idempotency.
|
||||||
|
|
||||||
Acceptance: rotating from the web console produces a reconnect using the new
|
Acceptance: rotating from the web console produces a reconnect using the new
|
||||||
token while the HTTP listener, collectors, and optional cluster stay running.
|
token while the HTTP listener, collectors, and optional cluster stay running.
|
||||||
|
|||||||
@@ -11,8 +11,10 @@ It does not yet execute `distributed_critical` checks, verify signed config,
|
|||||||
evaluate observation/region/notification quorum, encrypt snapshots, deliver a
|
evaluate observation/region/notification quorum, encrypt snapshots, deliver a
|
||||||
commit-backed outbox, or consume an external witness report.
|
commit-backed outbox, or consume an external witness report.
|
||||||
|
|
||||||
The hardcoded test-config endpoint and startup flag are development-only and
|
The former hardcoded test-config endpoint, startup flag, environment switch,
|
||||||
must stay disabled in production.
|
and production helper have been removed. Hardcoded config application now
|
||||||
|
exists only as an unexported `_test.go` helper and is absent from production
|
||||||
|
builds.
|
||||||
|
|
||||||
## Non-Negotiable Separation
|
## Non-Negotiable Separation
|
||||||
|
|
||||||
@@ -31,7 +33,8 @@ witness. It is never a Raft voter.
|
|||||||
|
|
||||||
## Topology
|
## Topology
|
||||||
|
|
||||||
- Production clusters have 3 or 5 voters, never an even count.
|
- Initial production clusters require exactly 3 voters. A future measured
|
||||||
|
five-voter profile may be introduced separately; never use an even count.
|
||||||
- Voters remain within a bounded-latency topology. Remote regions use observer
|
- Voters remain within a bounded-latency topology. Remote regions use observer
|
||||||
nodes unless measured RTT supports the configured election timeout.
|
nodes unless measured RTT supports the configured election timeout.
|
||||||
- Nodes may be `voter`, `observer`, or `voter+observer`.
|
- Nodes may be `voter`, `observer`, or `voter+observer`.
|
||||||
@@ -42,6 +45,12 @@ Single-voter bootstrap is temporary. The cluster must not execute customer
|
|||||||
critical checks until at least three voters are healthy and the signed
|
critical checks until at least three voters are healthy and the signed
|
||||||
observer set is committed.
|
observer set is committed.
|
||||||
|
|
||||||
|
Membership and mTLS identity are assigned by the control plane as defined in
|
||||||
|
[public-endpoint-and-identity.md](public-endpoint-and-identity.md). Production
|
||||||
|
Raft traffic uses `/raft` on each worker's external HTTPS origin through an
|
||||||
|
HTTP/1.1 upgrade-capable reverse proxy. Static peers and shared Basic auth are
|
||||||
|
lab compatibility only.
|
||||||
|
|
||||||
## Raft State
|
## Raft State
|
||||||
|
|
||||||
The FSM contains only:
|
The FSM contains only:
|
||||||
@@ -163,16 +172,18 @@ but cannot commit or fabricate incidents.
|
|||||||
|
|
||||||
## Ordered Implementation
|
## Ordered Implementation
|
||||||
|
|
||||||
1. Remove production exposure of debug config application.
|
1. [x] Remove production exposure of debug config application.
|
||||||
2. Complete versioned FSM types, command validation, and deterministic tests.
|
2. [ ] Complete versioned FSM types, command validation, and deterministic tests.
|
||||||
3. Add mTLS identity and safe one-claim cluster bootstrap.
|
3. [ ] Adapt the rbackup CA pattern into short-lived SAN-bound mTLS identity,
|
||||||
4. Add signed config and observer-set adoption.
|
rotation/revocation, and safe one-claim cluster bootstrap.
|
||||||
5. Add deterministic scheduler and checkexec bridge in shadow mode.
|
4. [ ] Add signed config and observer-set adoption.
|
||||||
6. Implement observation aggregation, incident policy, and idempotency.
|
5. [ ] Add deterministic scheduler and checkexec bridge in shadow mode.
|
||||||
7. Implement encrypted snapshots and restore/migration tests.
|
6. [ ] Implement observation aggregation, incident policy, and idempotency.
|
||||||
8. Add metadata outbox executor and failover-safe delivery.
|
7. [ ] Implement encrypted snapshots and restore/migration tests.
|
||||||
9. Add external witness, replay, metrics, and operational runbooks.
|
8. [ ] Add metadata outbox executor and failover-safe delivery.
|
||||||
10. Run synthetic 3/5-node fault campaigns before any customer check.
|
9. [ ] Add external witness, replay, metrics, and operational runbooks.
|
||||||
|
10. [ ] Run synthetic three-node HTTPS-proxy fault campaigns before any
|
||||||
|
customer check; define a separate five-node profile before testing it.
|
||||||
|
|
||||||
## Release Gates
|
## Release Gates
|
||||||
|
|
||||||
|
|||||||
@@ -14,8 +14,29 @@ Worker repository:
|
|||||||
- verify amd64 and arm64 image startup, Chromium availability, and version
|
- verify amd64 and arm64 image startup, Chromium availability, and version
|
||||||
metadata;
|
metadata;
|
||||||
- add a package/install smoke test for Docker and systemd artifacts;
|
- add a package/install smoke test for Docker and systemd artifacts;
|
||||||
|
- add the Docker/OpenSSH source-install matrix for Alpine, Ubuntu, and Arch from
|
||||||
|
[source-installation.md](source-installation.md), using Go 1.26 and the
|
||||||
|
remote default branch;
|
||||||
- document immutable SHA and release tags as production defaults.
|
- document immutable SHA and release tags as production defaults.
|
||||||
|
|
||||||
|
Source-install foundations landed:
|
||||||
|
|
||||||
|
- [x] reusable Docker/OpenSSH harness and distro fixtures for Alpine, Ubuntu,
|
||||||
|
and Arch (`internal/installer/harness`) with real SSH readiness and
|
||||||
|
reliable teardown, gated behind `RSMON_TEST_DOCKER` (`make test-ssh`);
|
||||||
|
- [x] pure distro/toolchain/source-install planning (`internal/sshinstall`):
|
||||||
|
os-release detection, package-manager/init resolution, pinned Go 1.26
|
||||||
|
toolchain with published SHA-256, and a plan the executor runs;
|
||||||
|
- [x] remote package install, Go download, clone, and build execution through
|
||||||
|
the SSH transport (`installer.SourceInstall`): prerequisite install,
|
||||||
|
SHA-256-verified Go toolchain, clone/update of the public repo, resolved
|
||||||
|
branch/commit record, and a staging build. Running service/config is not
|
||||||
|
touched (source-install work package 3);
|
||||||
|
- [x] atomic service activation, rollback, and failure-preservation tests over
|
||||||
|
SSH (source-install work package 4);
|
||||||
|
- [x] run the full source-install E2E matrix in CI (source-install work
|
||||||
|
package 5).
|
||||||
|
|
||||||
Gate: a push publishes `sha-<12>` and `latest` manifests for both platforms,
|
Gate: a push publishes `sha-<12>` and `latest` manifests for both platforms,
|
||||||
and a container remains healthy when the control plane is unavailable.
|
and a container remains healthy when the control plane is unavailable.
|
||||||
|
|
||||||
@@ -27,13 +48,19 @@ Worker repository:
|
|||||||
|
|
||||||
- wire local inventory and metrics collector lifecycle into web server start
|
- wire local inventory and metrics collector lifecycle into web server start
|
||||||
and shutdown;
|
and shutdown;
|
||||||
- validate full HTTP config, including `WORKER_URL`, in main startup;
|
- [x] validate full HTTP config, including the accepted `PUBLIC_URL`, in main
|
||||||
- make token rotation reconnect rather than stop the runner;
|
startup (milestone 1 of
|
||||||
|
[public-endpoint-and-identity.md](public-endpoint-and-identity.md): origin
|
||||||
|
shape, production HTTPS, legacy `WORKER_URL` fallback);
|
||||||
|
- [x] reconnect in memory on token rotation without stopping the runner;
|
||||||
|
- [x] resend bounded check/notification results after websocket reconnect;
|
||||||
- define process policy when the web listener exits unexpectedly;
|
- define process policy when the web listener exits unexpectedly;
|
||||||
- add WebSocket reconnect, drain, duplicate frame, and backpressure tests.
|
- add process-exit, bounded final-drain, duplicate-frame, and backpressure
|
||||||
|
coverage.
|
||||||
|
|
||||||
Gate: collectors populate real pages, rotation preserves all subsystems, and
|
Gate: collectors populate real pages, rotation preserves all subsystems, result
|
||||||
SIGTERM leaves no listener, collector, task, or SQLite goroutine behind.
|
resend remains bounded and idempotent, and SIGTERM leaves no listener,
|
||||||
|
collector, task, or SQLite goroutine behind.
|
||||||
|
|
||||||
## R2: Protocol And Credential Hardening
|
## R2: Protocol And Credential Hardening
|
||||||
|
|
||||||
@@ -67,7 +94,8 @@ Worker repository:
|
|||||||
- token handoff and revocation handling;
|
- token handoff and revocation handling;
|
||||||
- stale/expired signed-config behavior;
|
- stale/expired signed-config behavior;
|
||||||
- clear disabled/revoked UI state;
|
- clear disabled/revoked UI state;
|
||||||
- optional mTLS client identity.
|
- mandatory production mTLS client identity and control-plane-owned topology as
|
||||||
|
defined in [public-endpoint-and-identity.md](public-endpoint-and-identity.md).
|
||||||
|
|
||||||
Control-plane dependency:
|
Control-plane dependency:
|
||||||
|
|
||||||
@@ -136,13 +164,13 @@ State: Raft scaffold only.
|
|||||||
|
|
||||||
Worker repository:
|
Worker repository:
|
||||||
|
|
||||||
- secure bootstrap and mTLS identity;
|
- secure bootstrap, CA lifecycle, mTLS identity, and one-origin HTTPS transport;
|
||||||
- signed config adoption and observer-set versioning;
|
- signed config adoption and observer-set versioning;
|
||||||
- deterministic scheduler and check executor bridge;
|
- deterministic scheduler and check executor bridge;
|
||||||
- deterministic incident/quorum FSM;
|
- deterministic incident/quorum FSM;
|
||||||
- encrypted snapshots and restore;
|
- encrypted snapshots and restore;
|
||||||
- metadata outbox, witness, replay, and metrics;
|
- metadata outbox, witness, replay, and metrics;
|
||||||
- 3/5-node fault campaigns.
|
- three-node external-HTTPS fault campaigns.
|
||||||
|
|
||||||
Control-plane dependency:
|
Control-plane dependency:
|
||||||
|
|
||||||
|
|||||||
273
docs/install.md
Обычный файл
273
docs/install.md
Обычный файл
@@ -0,0 +1,273 @@
|
|||||||
|
# Installation
|
||||||
|
|
||||||
|
The `rsmon-worker install` subcommand turns a built binary (or a published
|
||||||
|
Docker image) into a running, enabled systemd service on a Linux host. It is
|
||||||
|
the supported way to deploy the worker: it writes the configuration, the
|
||||||
|
systemd unit, and the data directory, then starts the service.
|
||||||
|
|
||||||
|
> A Go SSH source installer (`rsmon-worker source-install`) builds the worker
|
||||||
|
> from source on a remote host over SSH - prerequisites, verified Go toolchain,
|
||||||
|
> clone/update of the public repository (resolved to the remote default branch
|
||||||
|
> unless pinned), resolved branch/commit record, and a staging build - without
|
||||||
|
> yet installing a service. See
|
||||||
|
> [source-installation.md](source-installation.md). Today the local
|
||||||
|
> `install` copies the binary you invoke it from (or pulls the `--image`
|
||||||
|
> digest), so build first with `make build` and run the resulting
|
||||||
|
> `./bin/rsmon-worker`.
|
||||||
|
|
||||||
|
## Requirements
|
||||||
|
|
||||||
|
- Linux with systemd.
|
||||||
|
- Root (`install must be run as root`). The worker runs as `root` so it can
|
||||||
|
reach the Docker socket for Compose discovery and use `CAP_NET_RAW` for ping
|
||||||
|
checks without extra setup.
|
||||||
|
- For the binary install: the worker binary you want to install.
|
||||||
|
- For the Docker install (`--docker`): the `docker` CLI and an immutable
|
||||||
|
`repository@sha256:<64 lowercase hex>` image digest.
|
||||||
|
|
||||||
|
Host dependencies for browser-backed HTTP checks (Debian/Ubuntu):
|
||||||
|
|
||||||
|
```bash
|
||||||
|
sudo apt-get update
|
||||||
|
sudo apt-get install -y ca-certificates chromium libcap2-bin tzdata
|
||||||
|
```
|
||||||
|
|
||||||
|
## Configuration sources
|
||||||
|
|
||||||
|
The installer reads the same environment variables the worker runtime reads.
|
||||||
|
Each variable is resolved with this precedence (highest first):
|
||||||
|
|
||||||
|
1. **Explicit flags** (`--url`, `--public-url`, `--token`, `--host`, `--port`,
|
||||||
|
`--login`, `--password`, `--name`).
|
||||||
|
2. **`--env-file`** — a strict, systemd-safe `KEY=VALUE` file (validated before
|
||||||
|
anything is written to disk).
|
||||||
|
3. **Process environment**, including a `.env` file in the working directory
|
||||||
|
(loaded automatically at startup via `godotenv`).
|
||||||
|
4. **Built-in defaults** (`RSMON_URL=https://rsmon.ru`, `WORKER_HOST=127.0.0.1`,
|
||||||
|
`WORKER_PORT=27401` for the primary instance).
|
||||||
|
|
||||||
|
`.env` files are loaded before the install command runs, so
|
||||||
|
`sudo rsmon-worker install` from a directory containing a `.env` picks up those
|
||||||
|
values automatically. To override a value, pass the matching flag.
|
||||||
|
|
||||||
|
### Variables
|
||||||
|
|
||||||
|
| Variable | Required | Default | Purpose |
|
||||||
|
| --- | --- | --- | --- |
|
||||||
|
| `RSMON_URL` | yes | `https://rsmon.ru` | Control-plane base URL. |
|
||||||
|
| `RSMON_TOKEN` | yes | none | Worker bearer token. |
|
||||||
|
| `WORKER_HOST` | no | `127.0.0.1` | Operator-console bind address. |
|
||||||
|
| `WORKER_PORT` | no | `27401` (primary) | Operator-console port. **Required** for named instances. |
|
||||||
|
| `PUBLIC_URL` | no | none | Advertised public origin: absolute http(s) URL with scheme and authority only (no userinfo, query, fragment, or path). Canonical name; `WORKER_URL` is a deprecated legacy alias read only during the bounded migration. |
|
||||||
|
| `WORKER_LOGIN` | no | `admin` (generated) | Operator-console basic-auth login. |
|
||||||
|
| `WORKER_PASSWORD` | no | generated | Operator-console basic-auth password. |
|
||||||
|
| `WORKER_COMPOSE_ENABLED` | no | feature default (on) | Enable Docker Compose discovery/management. |
|
||||||
|
| `WORKER_CLUSTER_*` | no | none | Optional Raft cluster (`ID`, `PORT`, `PEERS`, `DATA_DIR`, `ENABLED`). |
|
||||||
|
| `WORKER_RELEASE_URL` | no | none | Self-update release feed URL. |
|
||||||
|
|
||||||
|
`WORKER_LOGIN` and `WORKER_PASSWORD` must both be set or both be left empty. If
|
||||||
|
both are empty, the installer generates a random password (login `admin`),
|
||||||
|
writes it to the env file, and prints it once. Record it; the operator console
|
||||||
|
requires it for both the browser login and the `/web/api/*` HTTP basic-auth
|
||||||
|
endpoints.
|
||||||
|
|
||||||
|
`PUBLIC_URL` does not bind a listener or terminate TLS. It advertises the one
|
||||||
|
external origin used for the console, authenticated peer status, and planned
|
||||||
|
Raft `/raft` transport. It must be an absolute `http`/`https` URL with a scheme
|
||||||
|
and authority and nothing else; a path (other than `/`), userinfo, query, or
|
||||||
|
fragment is rejected at install time and at worker startup. `PUBLIC_URL` is the
|
||||||
|
canonical variable; the legacy `WORKER_URL` is still accepted for the bounded
|
||||||
|
migration defined in
|
||||||
|
[public-endpoint-and-identity.md](public-endpoint-and-identity.md), and is
|
||||||
|
dropped from a freshly written env file whenever `PUBLIC_URL` is also set.
|
||||||
|
The legacy `WORKER_URL` is held only to the tolerant absolute-URL check, so
|
||||||
|
shapes that previously installed keep working. In an explicitly production
|
||||||
|
environment (`DEPLOY_ENV=production`, or `RSMON_ENV`/`GO_ENV=production`) a
|
||||||
|
plain-HTTP `PUBLIC_URL` on a non-loopback host is rejected at startup; a legacy
|
||||||
|
`WORKER_URL` keeps the historical warn-only behavior. Both variables reject a
|
||||||
|
missing hostname, e.g. `https://:27401`.
|
||||||
|
|
||||||
|
Values must be systemd-safe: no whitespace, quotes, backslashes, or `$`
|
||||||
|
interpolation inside a value. This keeps the file unambiguous across systemd
|
||||||
|
`EnvironmentFile` and `docker --env-file`.
|
||||||
|
|
||||||
|
## Single-instance install (one worker per host)
|
||||||
|
|
||||||
|
The classic install uses the canonical paths and the default port 27401.
|
||||||
|
|
||||||
|
```bash
|
||||||
|
make build
|
||||||
|
printf '%s\n' 'YOUR_WORKER_TOKEN' > worker-token
|
||||||
|
chmod 600 worker-token
|
||||||
|
sudo ./bin/rsmon-worker install \
|
||||||
|
--token-file worker-token \
|
||||||
|
--port 27401 \
|
||||||
|
--password 'choose-a-console-password'
|
||||||
|
rm worker-token
|
||||||
|
```
|
||||||
|
|
||||||
|
If you keep configuration in a `.env` (recommended for repeatability):
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# .env
|
||||||
|
RSMON_URL=https://rsmon.ru
|
||||||
|
RSMON_TOKEN=...
|
||||||
|
WORKER_PORT=27401
|
||||||
|
WORKER_LOGIN=admin
|
||||||
|
WORKER_PASSWORD=...
|
||||||
|
```
|
||||||
|
|
||||||
|
```bash
|
||||||
|
sudo ./bin/rsmon-worker install
|
||||||
|
```
|
||||||
|
|
||||||
|
Installed layout:
|
||||||
|
|
||||||
|
| Path | Contents |
|
||||||
|
| --- | --- |
|
||||||
|
| `/usr/local/bin/rsmon-worker` | Worker binary. |
|
||||||
|
| `/etc/rsmon-worker/worker.env` | Configuration, mode `0600`. |
|
||||||
|
| `/var/lib/rsmon-worker/` | Data directory (SQLite, webapp state). |
|
||||||
|
| `/etc/systemd/system/rsmon-worker.service` | systemd unit. |
|
||||||
|
|
||||||
|
## Multi-instance install (several workers per host)
|
||||||
|
|
||||||
|
`--name` installs a co-located worker under `rsmon-worker-<name>` with its own
|
||||||
|
binary, config, data, systemd unit, and port. This is how you run a staging
|
||||||
|
build next to production, or isolate tenants on one host.
|
||||||
|
|
||||||
|
```bash
|
||||||
|
sudo ./bin/rsmon-worker install --name edge --port 27403 --password '...'
|
||||||
|
```
|
||||||
|
|
||||||
|
A named instance must have an explicit `WORKER_PORT` (the default 27401 belongs
|
||||||
|
to the primary). The name must be 1–32 chars, lowercase alphanumeric and
|
||||||
|
hyphens, starting and ending alphanumeric.
|
||||||
|
|
||||||
|
Named layout (for `--name edge`):
|
||||||
|
|
||||||
|
| Path | Contents |
|
||||||
|
| --- | --- |
|
||||||
|
| `/usr/local/bin/rsmon-worker-edge` | Binary. |
|
||||||
|
| `/etc/rsmon-worker-edge/worker.env` | Configuration, mode `0600`. |
|
||||||
|
| `/var/lib/rsmon-worker-edge/` | Data directory. |
|
||||||
|
| `/etc/systemd/system/rsmon-worker-edge.service` | systemd unit. |
|
||||||
|
|
||||||
|
Each instance is an independent service (`rsmon-worker.service`,
|
||||||
|
`rsmon-worker-edge.service`, …) and can be managed separately.
|
||||||
|
|
||||||
|
## What the installer does
|
||||||
|
|
||||||
|
For a binary install, `rsmon-worker install` performs these actions in order:
|
||||||
|
|
||||||
|
1. **Validates** the instance name, the `--env-file` (if any), and the resolved
|
||||||
|
`RSMON_URL`/`RSMON_TOKEN`. Nothing on the host changes before validation
|
||||||
|
passes.
|
||||||
|
2. **Resolves** every supported variable with the precedence above and, when
|
||||||
|
`WORKER_LOGIN`/`WORKER_PASSWORD` are both unset, generates a random console
|
||||||
|
password.
|
||||||
|
3. **Copies the binary** from the running executable (or `--binary`) to
|
||||||
|
`/usr/local/bin/rsmon-worker[-<name>]` with an atomic rename.
|
||||||
|
4. **Creates** the data directory (`/var/lib/rsmon-worker[-<name>]/webapp`) and
|
||||||
|
the config directory (`/etc/rsmon-worker[-<name>]`, mode `0750`).
|
||||||
|
5. **Writes the env file** to `<config dir>/worker.env` (mode `0600`), in a
|
||||||
|
stable canonical order.
|
||||||
|
6. **Writes the systemd unit** to
|
||||||
|
`/etc/systemd/system/rsmon-worker[-<name>].service`.
|
||||||
|
7. Runs `systemctl daemon-reload`, `systemctl enable`, and, unless `--no-start`,
|
||||||
|
`systemctl restart` followed by `systemctl is-active --quiet` to confirm the
|
||||||
|
service came up.
|
||||||
|
8. **Prints a summary**: unit name, binary, env file, data dir, console URL,
|
||||||
|
status, and the generated password if one was created.
|
||||||
|
|
||||||
|
### The generated systemd unit
|
||||||
|
|
||||||
|
The unit is a hardened `Type=simple` root service ordered after
|
||||||
|
`network-online.target` and `docker.service`:
|
||||||
|
|
||||||
|
- `ExecStart=/usr/local/bin/rsmon-worker[-<name>]`
|
||||||
|
- `EnvironmentFile=/etc/rsmon-worker[-<name>]/worker.env`
|
||||||
|
- `Environment=HOME=/var/lib/rsmon-worker[-<name>]`
|
||||||
|
- `Environment=RSMON_WEBAPP_DATA_DIR=/var/lib/rsmon-worker[-<name>]/webapp`
|
||||||
|
- `AmbientCapabilities=CAP_NET_RAW` / `CapabilityBoundingSet=CAP_NET_RAW` for
|
||||||
|
ICMP checks.
|
||||||
|
- `PrivateTmp`, `ProtectHome`, `ProtectSystem=full`, and
|
||||||
|
`ReadWritePaths=<data dir>` to constrain writes.
|
||||||
|
- `Restart=always`, `RestartSec=5s`.
|
||||||
|
|
||||||
|
## Docker install
|
||||||
|
|
||||||
|
Installs a systemd unit that runs the published image instead of a local
|
||||||
|
binary. Requires an immutable digest; mutable tags are rejected.
|
||||||
|
|
||||||
|
```bash
|
||||||
|
sudo ./bin/rsmon-worker install --docker \
|
||||||
|
--image 'reg.rsxx.ru/rsmon/rsmon-worker@sha256:<64-hex-digest>' \
|
||||||
|
--token-file worker-token
|
||||||
|
```
|
||||||
|
|
||||||
|
`--name` works with `--docker` too; the container and volume are namespaced by
|
||||||
|
instance (`rsmon-worker-edge`, `rsmon-worker-data-edge`). The container always
|
||||||
|
mounts its data volume at the in-image `/var/lib/rsmon-worker`.
|
||||||
|
|
||||||
|
## Options reference
|
||||||
|
|
||||||
|
```
|
||||||
|
rsmon-worker install [--token TOKEN|--token-file FILE|--env-file FILE]
|
||||||
|
[--name NAME] [--port PORT] [options]
|
||||||
|
```
|
||||||
|
|
||||||
|
| Flag | Purpose |
|
||||||
|
| --- | --- |
|
||||||
|
| `--token`, `--api-key` | Worker API token (`RSMON_TOKEN`). |
|
||||||
|
| `--token-file` | File containing the worker token (avoids shell history). |
|
||||||
|
| `--env-file` | Strict worker env file; validated then used as the config source. |
|
||||||
|
| `--url` | Control-plane URL (`RSMON_URL`). |
|
||||||
|
| `--public-url` | Advertised public origin (`PUBLIC_URL`; scheme + host, no path). |
|
||||||
|
| `--host` | Console bind address (`WORKER_HOST`). |
|
||||||
|
| `--port` | Console port (`WORKER_PORT`; required with `--name`). |
|
||||||
|
| `--login` | Console login (`WORKER_LOGIN`). |
|
||||||
|
| `--password`, `--password-file` | Console password, or a file containing it. |
|
||||||
|
| `--name` | Instance name for a co-located worker. |
|
||||||
|
| `--binary` | Binary to install (default: this executable). |
|
||||||
|
| `--docker` | Install the prebuilt Docker image (requires `--image`). |
|
||||||
|
| `--image` | Immutable `repository@sha256:<64 hex>` digest. |
|
||||||
|
| `--no-start` | Enable without starting. |
|
||||||
|
|
||||||
|
`--token`/`--token-file` and `--password`/`--password-file` are mutually
|
||||||
|
exclusive within each pair; combining a direct secret with its file form is an
|
||||||
|
error.
|
||||||
|
|
||||||
|
## Post-install operations
|
||||||
|
|
||||||
|
```bash
|
||||||
|
systemctl status rsmon-worker[-<name>]
|
||||||
|
journalctl -u rsmon-worker[-<name>] -f
|
||||||
|
sudo systemctl restart rsmon-worker[-<name>]
|
||||||
|
```
|
||||||
|
|
||||||
|
The public liveness endpoint is `GET /healthz`; probe it with
|
||||||
|
`rsmon-worker liveness`. The operator console is at `http://127.0.0.1:<port>`
|
||||||
|
(loopback by default) and exposes the JSON API under `/web/api/*` using HTTP
|
||||||
|
basic auth.
|
||||||
|
|
||||||
|
### Uninstall
|
||||||
|
|
||||||
|
There is no `uninstall` subcommand yet. To remove an instance manually:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
sudo systemctl disable --now rsmon-worker-<name>
|
||||||
|
sudo rm /etc/systemd/system/rsmon-worker-<name>.service
|
||||||
|
sudo rm -rf /etc/rsmon-worker-<name> /var/lib/rsmon-worker-<name> /usr/local/bin/rsmon-worker-<name>
|
||||||
|
sudo systemctl daemon-reload
|
||||||
|
```
|
||||||
|
|
||||||
|
## Security notes
|
||||||
|
|
||||||
|
- Keep `/etc/rsmon-worker[-<name>]/worker.env` mode `0600`; the installer writes
|
||||||
|
it that way.
|
||||||
|
- Prefer `--token-file`/`--password-file` or a `.env` over passing secrets as
|
||||||
|
flags, which can leak through shell history and process inspection.
|
||||||
|
- Bind the console to loopback (`WORKER_HOST=127.0.0.1`, the default) or front
|
||||||
|
it with an authenticated TLS reverse proxy.
|
||||||
|
- Each worker should use its own control-plane token.
|
||||||
@@ -8,7 +8,7 @@ plane validates identity, applies account scope, reconciles lifecycle, and
|
|||||||
persists the inventory projection.
|
persists the inventory projection.
|
||||||
|
|
||||||
The control-plane inventory entities remain `Server`, `ServerIp`, `Site`,
|
The control-plane inventory entities remain `Server`, `ServerIp`, `Site`,
|
||||||
`Deployment`, `Domain`, `Repo`, and `site_repos`. Their API and rstuff stream
|
`Deployment`, `Domain`, `Repo`, and `site_repos`. Their API and legacy stream
|
||||||
projection are control-plane concerns. This repository owns only local
|
projection are control-plane concerns. This repository owns only local
|
||||||
discovery and worker-originated reports.
|
discovery and worker-originated reports.
|
||||||
|
|
||||||
@@ -130,7 +130,7 @@ successful section can mark previously observed entities missing.
|
|||||||
- Do not guess a primary address; report interface and route metadata so the
|
- Do not guess a primary address; report interface and route metadata so the
|
||||||
control plane can apply policy.
|
control plane can apply policy.
|
||||||
|
|
||||||
## Relationship To deploymentd And rstuff
|
## Relationship To deploymentd And RSLogin
|
||||||
|
|
||||||
Worker reports and deploymentd ingest are complementary producers of the same
|
Worker reports and deploymentd ingest are complementary producers of the same
|
||||||
control-plane projection. During migration:
|
control-plane projection. During migration:
|
||||||
@@ -140,8 +140,9 @@ control-plane projection. During migration:
|
|||||||
- every row records source and source report ID;
|
- every row records source and source report ID;
|
||||||
- reconciliation occurs per source and section to prevent one producer from
|
- reconciliation occurs per source and section to prevent one producer from
|
||||||
deleting another producer's observations;
|
deleting another producer's observations;
|
||||||
- the rstuff `rsmon.inventory.v1` stream remains a control-plane output/input,
|
- the former rstuff stream direction is superseded; RSMon owns inventory and
|
||||||
not a worker protocol.
|
RSLogin supplies trusted project and SSH-access input through a separate
|
||||||
|
control-plane integration, never a worker protocol.
|
||||||
|
|
||||||
## Delivery Phases
|
## Delivery Phases
|
||||||
|
|
||||||
|
|||||||
@@ -55,19 +55,22 @@ Bootstrap replay, expiry, worker-ID mismatch, or account mismatch fails closed.
|
|||||||
|
|
||||||
## Token Rotation
|
## Token Rotation
|
||||||
|
|
||||||
Rotation is a two-token handoff:
|
The current worker receives a replacement token in memory. In-memory reconnect
|
||||||
|
rotation is implemented: the control plane
|
||||||
|
invalidates the old token immediately, then `Runner.RotateToken` closes only
|
||||||
|
the active control-plane connection and reconnects with the replacement token.
|
||||||
|
It does not stop the runner or its web, inventory, metrics, or cluster
|
||||||
|
subsystems. There is no durable token write, overlap, rollback, or rotation
|
||||||
|
acknowledgement contract yet; persistent replacement-token failures can require
|
||||||
|
operator action.
|
||||||
|
|
||||||
1. Control plane issues a replacement token and keeps the old token valid for a
|
If a websocket write fails after the worker has dequeued a check or notification
|
||||||
short bounded overlap.
|
result, it resends that envelope after reconnect. Delivery is therefore
|
||||||
2. Worker atomically writes the replacement secret.
|
at-least-once; the control plane must deduplicate by the leased job and lease
|
||||||
3. Worker closes only the active control-plane connection and reconnects.
|
token before applying a resent result. Failed server-metric snapshots are
|
||||||
4. Successful authentication acknowledges rotation; control plane revokes the
|
dropped because they have no idempotency key and the next periodic collection
|
||||||
old token.
|
replaces them. Metric snapshots are bound to the active control connection;
|
||||||
5. Failure retains the old token until overlap expires and emits an operator
|
snapshots collected while disconnected or for an older connection are dropped.
|
||||||
warning.
|
|
||||||
|
|
||||||
The current `Runner.RotateToken` terminates the runner by closing its stop
|
|
||||||
channel. This must be fixed before claiming unattended rotation.
|
|
||||||
|
|
||||||
## Runtime Config Authentication
|
## Runtime Config Authentication
|
||||||
|
|
||||||
@@ -85,6 +88,11 @@ signature failure, downgrade, expiry, account change, and unknown critical
|
|||||||
fields. Credentials remain in memory and are cleared when their signed scope
|
fields. Credentials remain in memory and are cleared when their signed scope
|
||||||
expires.
|
expires.
|
||||||
|
|
||||||
|
Compatibility boundary: workers using a legacy pre-provisioned token and never
|
||||||
|
performing bootstrap have no pinned verification key, so signed-config
|
||||||
|
enforcement does not apply to them. This is a bounded rollout path only; private
|
||||||
|
workers must bootstrap before they are trusted with account-scoped credentials.
|
||||||
|
|
||||||
## Public Checks
|
## Public Checks
|
||||||
|
|
||||||
Cross-account public-check execution is not the same as normal private scope.
|
Cross-account public-check execution is not the same as normal private scope.
|
||||||
@@ -113,8 +121,26 @@ type, control host access:
|
|||||||
- Compose mutation: separate high-risk capability, disabled by default;
|
- Compose mutation: separate high-risk capability, disabled by default;
|
||||||
- Raft voter: durable fsync-capable cluster data directory and mTLS transport.
|
- Raft voter: durable fsync-capable cluster data directory and mTLS transport.
|
||||||
|
|
||||||
`CAP_NET_RAW` is granted only for ping/traceroute features. The worker remains
|
The simple systemd installer currently runs the worker as root, matching the
|
||||||
unprivileged otherwise.
|
minimal host-install model. Docker runs with the image's unprivileged user and
|
||||||
|
adds `NET_RAW` for ping/traceroute. A future hardened systemd profile can use a
|
||||||
|
dedicated user and narrow capabilities when host inventory requirements are
|
||||||
|
finalized.
|
||||||
|
|
||||||
|
The standalone binary provides two systemd installation paths:
|
||||||
|
|
||||||
|
- `rsmon-worker install` installs the current binary locally and writes the
|
||||||
|
worker URL/token to a root-owned mode-0600 environment file; `--docker`
|
||||||
|
installs a systemd-managed prebuilt image instead;
|
||||||
|
- `rsmon-worker deploy` verifies an SSH host key, uploads the binary and a
|
||||||
|
temporary mode-0600 environment file, and invokes `install` remotely;
|
||||||
|
`--docker` uploads only the environment and unit, then pulls an explicitly
|
||||||
|
supplied immutable image digest on the target.
|
||||||
|
|
||||||
|
Both default to `https://rsmon.ru` and accept `--token-file` for automation.
|
||||||
|
Direct secret flags are supported but can be visible in process listings; file
|
||||||
|
options are preferred. SSH login credentials, sudo credentials, and the worker
|
||||||
|
token remain separate.
|
||||||
|
|
||||||
## Worker Self-Monitoring
|
## Worker Self-Monitoring
|
||||||
|
|
||||||
@@ -126,22 +152,25 @@ monitor execution.
|
|||||||
|
|
||||||
## Implementation Work Packages
|
## Implementation Work Packages
|
||||||
|
|
||||||
1. Add signed account/config identity to `internal/wire` and runner state.
|
1. [x] Add signed account/config identity to `internal/wire` and runner state.
|
||||||
2. Validate task account and credential scope locally before dispatch.
|
2. [x] Validate task account scope locally before dispatch.
|
||||||
3. Implement one-time bootstrap and atomic token storage/rotation.
|
3. [x] Implement in-memory reconnect token rotation without stopping worker
|
||||||
4. Add worker disable/revoke behavior and visible stale-config state.
|
subsystems.
|
||||||
5. Add mTLS as an optional first transport, then require it for Raft clusters.
|
4. [x] Implement one-time bootstrap, durable token storage, and rotation
|
||||||
6. Add public-task grant and SSRF-safe executor only after private isolation is
|
acknowledgement.
|
||||||
|
5. [ ] Add worker disable/revoke behavior and visible stale-config state.
|
||||||
|
6. [ ] Add mTLS as an optional first transport, then require it for Raft clusters.
|
||||||
|
7. [ ] Add public-task grant and SSRF-safe executor only after private isolation is
|
||||||
proven.
|
proven.
|
||||||
|
|
||||||
## Acceptance Tests
|
## Acceptance Tests
|
||||||
|
|
||||||
- A private worker never leases or executes another account's normal task.
|
- [ ] A private worker never leases or executes another account's normal task.
|
||||||
- A forged capability or account field cannot widen scope.
|
- [ ] A forged capability or account field cannot widen scope.
|
||||||
- Invalid, expired, downgraded, or differently scoped signed config is rejected.
|
- [ ] Invalid, expired, downgraded, or differently scoped signed config is rejected.
|
||||||
- Bootstrap tokens are single-use and absent from disk after exchange.
|
- [ ] Bootstrap tokens are single-use and absent from disk after exchange.
|
||||||
- Rotation reconnects without stopping web, inventory, metrics, or cluster.
|
- [x] Rotation reconnects without stopping web, inventory, metrics, or cluster.
|
||||||
- Credential snapshots contain only the worker account and allowed methods.
|
- [ ] Credential snapshots contain only the worker account and allowed methods.
|
||||||
- Public execution cannot reach loopback, link-local, RFC1918, metadata, Unix
|
- [ ] Public execution cannot reach loopback, link-local, RFC1918, metadata, Unix
|
||||||
sockets, or a private redirect target.
|
sockets, or a private redirect target.
|
||||||
- Revocation prevents reconnect and clears in-memory credentials.
|
- [ ] Revocation prevents reconnect and clears in-memory credentials.
|
||||||
|
|||||||
143
docs/public-endpoint-and-identity.md
Обычный файл
143
docs/public-endpoint-and-identity.md
Обычный файл
@@ -0,0 +1,143 @@
|
|||||||
|
# Public Endpoint, Peer Identity, And Raft Transport
|
||||||
|
|
||||||
|
## Status
|
||||||
|
|
||||||
|
Accepted target architecture. Ordinary peer status checks already support an
|
||||||
|
external HTTPS worker URL. Raft currently uses a separate plaintext listener,
|
||||||
|
static peers, and shared Basic authentication; it does not yet meet this plan.
|
||||||
|
|
||||||
|
Milestone 1 (worker side) is implemented: the worker configures `PUBLIC_URL`
|
||||||
|
as the canonical advertised origin with the legacy `WORKER_URL` accepted only
|
||||||
|
for a bounded migration, and validates the origin shape at startup and install
|
||||||
|
time. On the wire the worker consumes the accepted endpoint from
|
||||||
|
`wire.WorkerInit` (`public_url`, falling back to the legacy `url` field) and
|
||||||
|
rejects unusable values. Transmitting the proposed URL during registration is
|
||||||
|
the pending RSMon control-plane counterpart: it must populate `public_url` in
|
||||||
|
`WorkerInit` and accept `RegisterRequest.public_url`; until then the worker
|
||||||
|
reads whichever field the control plane sends.
|
||||||
|
|
||||||
|
## One Worker, One Public URL
|
||||||
|
|
||||||
|
Every worker configures one absolute `PUBLIC_URL`, for example:
|
||||||
|
|
||||||
|
```text
|
||||||
|
https://worker-1.example.net
|
||||||
|
```
|
||||||
|
|
||||||
|
The worker proposes this URL during registration/configuration. The control
|
||||||
|
plane validates and canonicalizes it, persists the accepted value, and returns
|
||||||
|
it in signed init/config. The URL has scheme and authority only: no userinfo,
|
||||||
|
query, fragment, or ambiguous path. Public production URLs use HTTPS.
|
||||||
|
|
||||||
|
The control plane owns worker ID, account, region, cluster ID, membership,
|
||||||
|
voter/observer role, peer generation, and topology. A worker cannot gain trust
|
||||||
|
or Raft membership by advertising an endpoint.
|
||||||
|
|
||||||
|
## Routes On The Origin
|
||||||
|
|
||||||
|
- `/healthz`: process liveness;
|
||||||
|
- `/api/peer/status`: authenticated peer/control-plane reachability and Raft
|
||||||
|
summary;
|
||||||
|
- `/raft`: Hashicorp Raft HTTP/1.1 upgrade transport and membership requests;
|
||||||
|
- `/web/*` and `/web/api/*`: operator console.
|
||||||
|
|
||||||
|
TLS termination belongs to Traefik or nginx. The worker may keep internal HTTP
|
||||||
|
listeners on loopback/private addresses. `PUBLIC_URL` is advertisement, not a
|
||||||
|
request to bind the worker process or provision a public certificate.
|
||||||
|
|
||||||
|
For `/raft`, proxies must preserve path, query, Host, authorization/client
|
||||||
|
identity, `Connection`, and `Upgrade`; use HTTP/1.1, disable response buffering,
|
||||||
|
and allow long-lived upgraded connections. nginx requires explicit upgrade
|
||||||
|
headers. Tests must exercise this proxy path rather than only direct TCP.
|
||||||
|
|
||||||
|
## In-Memory Peer Database
|
||||||
|
|
||||||
|
Signed control-plane config supplies a monotonic topology generation and peers
|
||||||
|
scoped to one cluster/trust boundary. Each entry includes worker ID,
|
||||||
|
`PUBLIC_URL`, role, region, certificate identity, membership state, and expiry.
|
||||||
|
|
||||||
|
The worker replaces its in-memory peer database atomically when a newer valid
|
||||||
|
generation arrives. Removed or expired peers stop contributing immediately.
|
||||||
|
Periodic concurrent HTTP checks call each peer's `/api/peer/status`, validate
|
||||||
|
the TLS/client identity and response worker ID, and retain bounded observations.
|
||||||
|
Reachability never grants membership.
|
||||||
|
|
||||||
|
The response distinguishes process health, control-plane ping duration/result,
|
||||||
|
observation time, Raft role/term/leader, applied/commit indexes, and transport
|
||||||
|
errors. The worker console presents the same bounded per-peer state.
|
||||||
|
|
||||||
|
## Bind And Advertise Separation
|
||||||
|
|
||||||
|
Raft requires separate local bind and advertised addresses. The local listener
|
||||||
|
may be `127.0.0.1:37401`; the advertised endpoint derives from
|
||||||
|
`PUBLIC_URL + /raft`. Hashicorp Raft membership stores the advertised address,
|
||||||
|
never the loopback bind.
|
||||||
|
|
||||||
|
Outbound Raft transport performs TLS using the public hostname and worker
|
||||||
|
certificate. The current `rafthttp.NewDialTCP` and shared Basic-auth injection
|
||||||
|
must be replaced. Static `WORKER_CLUSTER_PEERS` remains a local lab fallback,
|
||||||
|
not production discovery.
|
||||||
|
|
||||||
|
## Certificate Authority
|
||||||
|
|
||||||
|
Adapt the proven CA shape from `/data/_backup/rbackup-server`, particularly its
|
||||||
|
explicit CA pool, required client-certificate verification, and certificate-to-
|
||||||
|
database identity lookup. Do not copy its tracked keys, ten-year certificates,
|
||||||
|
CN-only identity, inactive CRL, or server-generated private-key delivery.
|
||||||
|
|
||||||
|
The control plane owns a worker-cluster CA separate from public web
|
||||||
|
certificates. Requirements:
|
||||||
|
|
||||||
|
- worker-generated key and CSR;
|
||||||
|
- worker and cluster identity in a verified URI SAN;
|
||||||
|
- short-lived leaf certificates with automatic renewal;
|
||||||
|
- encrypted CA private key and documented backup/custody;
|
||||||
|
- issuance and revocation audit;
|
||||||
|
- emergency revocation plus short expiry as the normal revocation bound;
|
||||||
|
- rejection of expired, revoked, wrong-cluster, or SAN-mismatched peers.
|
||||||
|
|
||||||
|
One-time enrollment tokens authorize a CSR exactly once. They do not become
|
||||||
|
ongoing peer credentials. Browser traffic does not require a client certificate;
|
||||||
|
the reverse proxy applies mTLS policy to peer/Raft routes or forwards a verified
|
||||||
|
client identity through a tightly controlled internal boundary.
|
||||||
|
|
||||||
|
## Production Membership
|
||||||
|
|
||||||
|
Production critical-check clusters require exactly three voters initially.
|
||||||
|
Single-node bootstrap and two-node clusters are lab/bootstrap states and cannot
|
||||||
|
execute customer critical checks. Membership changes are leader-mediated and
|
||||||
|
control-plane-authorized. The control plane is an external witness, never a
|
||||||
|
voter.
|
||||||
|
|
||||||
|
## Implementation Order
|
||||||
|
|
||||||
|
1. Add `PUBLIC_URL` wire/config fields while accepting legacy `WORKER_URL` only
|
||||||
|
for a bounded migration.
|
||||||
|
|
||||||
|
Worker side implemented: `PUBLIC_URL` is canonical, `WORKER_URL` is a
|
||||||
|
deprecated fallback with a startup warning, `internal/wire` carries
|
||||||
|
`public_url` on `WorkerInit` (keeping `url` for compatibility), and
|
||||||
|
startup/install enforce the strict origin shape for `PUBLIC_URL` while
|
||||||
|
tolerating legacy `WORKER_URL` shapes. Control-plane counterpart: read
|
||||||
|
`RegisterRequest.public_url` when worker-initiated registration is wired,
|
||||||
|
populate `public_url` (not `url`) in `WorkerInit`, and persist the accepted
|
||||||
|
origin. Until then the worker consumes whichever of `public_url`/`url` the
|
||||||
|
control plane sends.
|
||||||
|
2. Validate ownership/reachability and return accepted signed configuration.
|
||||||
|
3. Add scoped, versioned in-memory peer topology and concurrent health probes.
|
||||||
|
4. Extend status APIs/UI with networking, control-plane RTT, and Raft state.
|
||||||
|
5. Implement CA storage, CSR enrollment, renewal, revocation, and audit.
|
||||||
|
6. Split Raft bind/advertise addresses and add outbound TLS dialing.
|
||||||
|
7. Route `/raft` through the shared public origin and reverse-proxy contract.
|
||||||
|
8. Replace static production peers with control-plane-managed membership.
|
||||||
|
9. Run a three-voter TLS-proxy failover/partition campaign.
|
||||||
|
|
||||||
|
## Release Gates
|
||||||
|
|
||||||
|
- A peer cannot impersonate another worker or join another cluster.
|
||||||
|
- An invalid/replayed enrollment token issues no certificate.
|
||||||
|
- Revoked, expired, or SAN-mismatched certificates fail before Raft handling.
|
||||||
|
- Three workers join through external HTTPS origins and elect one leader.
|
||||||
|
- Killing one voter preserves quorum and commits through the proxy path.
|
||||||
|
- The UI reports each peer, transport state, control-plane RTT, role, term,
|
||||||
|
leader, and replication lag without exposing credentials.
|
||||||
58
docs/sessions/2026-07-19-standalone-deployment.md
Обычный файл
58
docs/sessions/2026-07-19-standalone-deployment.md
Обычный файл
@@ -0,0 +1,58 @@
|
|||||||
|
# Session: Standalone Worker Deployment
|
||||||
|
|
||||||
|
Date: 2026-07-19
|
||||||
|
|
||||||
|
## Goal
|
||||||
|
|
||||||
|
Make `rsmon-worker` independently installable and deployable without requiring
|
||||||
|
the main `rsmon` repository. Keep direct binary plus systemd installation as the
|
||||||
|
default and offer the prebuilt Docker image as an explicit alternative.
|
||||||
|
|
||||||
|
## Completed
|
||||||
|
|
||||||
|
- Audited build and runtime paths for dependencies on `/data/_devops/rsmon` and
|
||||||
|
`/data/rsmon`.
|
||||||
|
- Removed the obsolete local certificate-bundle fallback.
|
||||||
|
- Switched the Dockerfile to public `golang:1-trixie` and `debian:13-slim`
|
||||||
|
bases.
|
||||||
|
- Implemented local binary and Docker installation in `internal/installer`.
|
||||||
|
- Implemented SSH deployment with key, password, passphrase, known-hosts, and
|
||||||
|
fingerprint support.
|
||||||
|
- Added `install` and `deploy` command handling and secret-file options to the
|
||||||
|
CLI.
|
||||||
|
- Added and updated installer tests, systemd packaging, deployment docs, and
|
||||||
|
the legacy installation wrapper.
|
||||||
|
- Reviewed the implementation for release blockers and secret exposure.
|
||||||
|
- Committed and pushed the work to `origin/master` as `3256dcd`.
|
||||||
|
|
||||||
|
## Decisions
|
||||||
|
|
||||||
|
- The default service uses `Type=simple`, runs as `root`, and restarts only on
|
||||||
|
failure.
|
||||||
|
- Docker deployment pulls an explicitly selected immutable image digest on the
|
||||||
|
target host instead of uploading a local image or binary. The original
|
||||||
|
mutable `latest` default was superseded by a supply-chain hardening change.
|
||||||
|
- The existing source-available license remains unchanged until public-release
|
||||||
|
licensing is decided.
|
||||||
|
- Worker credentials are supplied at deployment time and are not stored in the
|
||||||
|
repository.
|
||||||
|
|
||||||
|
## Verification
|
||||||
|
|
||||||
|
- `make test`
|
||||||
|
- `make build`
|
||||||
|
- `go vet ./cmd/rsmon-worker ./internal/installer`
|
||||||
|
- `go mod verify`
|
||||||
|
- Docker image pull and `--version` execution
|
||||||
|
- Compose configuration with a temporary environment fixture
|
||||||
|
- Live WebSocket connection, configuration receipt, job execution, and result
|
||||||
|
reporting
|
||||||
|
|
||||||
|
## Operational State
|
||||||
|
|
||||||
|
- A local worker was left running as `./bin/rsmon-worker --no-web`, PID
|
||||||
|
`211207`, with logs in `/tmp/rsmon-worker-local.log`.
|
||||||
|
- The worker received configuration as worker ID `test` and continued reporting
|
||||||
|
results after automatic reconnections caused by periodic WebSocket EOFs.
|
||||||
|
- Pushing `master` triggered `.github/workflows/docker.yml`; workflow completion
|
||||||
|
was not checked during the session.
|
||||||
595
docs/source-installation.md
Обычный файл
595
docs/source-installation.md
Обычный файл
@@ -0,0 +1,595 @@
|
|||||||
|
# SSH Source Installation Plan
|
||||||
|
|
||||||
|
## Status
|
||||||
|
|
||||||
|
In progress. Work packages 1-5 are implemented (work package 5 is the
|
||||||
|
Docker/OpenSSH E2E matrix wired into CI):
|
||||||
|
|
||||||
|
- `internal/installer/harness` builds and runs real OpenSSH containers
|
||||||
|
for Alpine, Ubuntu, and Arch, waits for real network readiness, captures
|
||||||
|
the server host key into a temp `known_hosts` file, and tears the
|
||||||
|
container, network, per-instance fixture image tag, and temp dir down
|
||||||
|
reliably. It never mocks SSH and connects with the
|
||||||
|
`golang.org/x/crypto/ssh` library and known_hosts verification
|
||||||
|
semantics the installer's `deploy` path relies on.
|
||||||
|
- `internal/sshinstall` resolves a remote host's distro, package manager,
|
||||||
|
and init system from `/etc/os-release`, plans the pinned Go 1.26
|
||||||
|
toolchain (published SHA-256) for the remote architecture, and produces
|
||||||
|
a pure source-install plan. It executes nothing.
|
||||||
|
- `installer.SourceInstall` (work package 3) executes the plan through
|
||||||
|
the same SSH transport, authentication, and host-key verification the
|
||||||
|
`deploy` command uses. It installs the minimal package prerequisites,
|
||||||
|
downloads and SHA-256-verifies the pinned Go toolchain before
|
||||||
|
extraction, clones/updates the public repository, checks out the
|
||||||
|
resolved branch, records the resolved branch and commit, and builds the
|
||||||
|
worker to a staging path.
|
||||||
|
- Work package 4 (`installer.SourceInstall` with `Activation` enabled)
|
||||||
|
atomically installs the staged binary, validated environment, data
|
||||||
|
directory, and the detected init's service definition; starts/restarts
|
||||||
|
the worker; verifies the process and `/healthz`; and rolls back to the
|
||||||
|
prior working install on any activation/start/health failure. Reruns
|
||||||
|
are idempotent with exactly one running worker and no leaked temp
|
||||||
|
files. The CLI runs the full flow by default and accepts
|
||||||
|
`--no-activate` (staging only) and `--no-start`.
|
||||||
|
|
||||||
|
The current Go installer can also upload a binary or deploy an immutable
|
||||||
|
Docker image over SSH (`deploy`). Source installs build remotely, then
|
||||||
|
activate the staged build atomically with rollback. Existing tests are
|
||||||
|
unit tests plus the harness tests that run against live OpenSSH
|
||||||
|
containers when explicitly enabled.
|
||||||
|
|
||||||
|
## Initial Platform Scope
|
||||||
|
|
||||||
|
The first source installer supports Linux and is validated on Alpine, Ubuntu,
|
||||||
|
and Arch Linux containers. CentOS-family support follows after its package and
|
||||||
|
service differences are implemented. Windows and macOS remain later platform
|
||||||
|
work despite the worker being written in Go.
|
||||||
|
|
||||||
|
Use `reg.rsxx.ru` image mirrors where available. Tests must not depend on Docker
|
||||||
|
Hub when a local mirror exists.
|
||||||
|
|
||||||
|
## Source Install Flow
|
||||||
|
|
||||||
|
The Go CLI connects through `golang.org/x/crypto/ssh` using existing key,
|
||||||
|
passphrase, password, sudo-password, known-hosts, and pinned-fingerprint support.
|
||||||
|
It then:
|
||||||
|
|
||||||
|
1. detects supported OS, architecture, package manager, init system, and
|
||||||
|
privilege path;
|
||||||
|
2. installs only required packages (`git`, CA certificates, download/archive
|
||||||
|
tools); it does not install `build-essential` or a C compiler unless a
|
||||||
|
detected dependency requires CGO;
|
||||||
|
3. downloads the pinned Go 1.26 toolchain for the detected architecture and
|
||||||
|
verifies the published SHA-256;
|
||||||
|
4. clones `https://rocketgit.ru/rsmon/worker.git` or updates an existing clone;
|
||||||
|
5. checks out the resolved branch (the pinned branch when one is configured,
|
||||||
|
otherwise the remote's default branch) and records the resolved commit;
|
||||||
|
6. builds a reproducible worker binary with the repository build flags;
|
||||||
|
7. atomically installs the binary, validated environment, data directory, and
|
||||||
|
service definition;
|
||||||
|
8. starts the service and verifies process status and `/healthz`.
|
||||||
|
Repository, branch, Go version, checksum source, build directory, and Go module
|
||||||
|
proxy may be configurable, but production output records their resolved values.
|
||||||
|
The default repository is publicly readable and requires no source credential.
|
||||||
|
|
||||||
|
## Work Package 3: Remote Execution To A Staging Path
|
||||||
|
|
||||||
|
Work package 3 is `installer.SourceInstall` in
|
||||||
|
`internal/installer/sourceinstall.go`. It reuses the `deploy` command's
|
||||||
|
`SSHOptions` (authentication, sudo password, known-hosts and fingerprint
|
||||||
|
verification) and runs every remote step with the same privilege path
|
||||||
|
(root, passwordless sudo, or `sudo -S -p ''` with the password delivered
|
||||||
|
only over stdin). Steps 1-6 of the flow above are implemented; step 7
|
||||||
|
(atomic install) and step 8 (start + verify) are work package 4.
|
||||||
|
|
||||||
|
Per step:
|
||||||
|
|
||||||
|
- **Prerequisite install.** `packageScript` renders the distro's
|
||||||
|
idempotent command (`apk add --no-cache`, `apt-get update` +
|
||||||
|
`apt-get install -y --no-install-recommends`, `pacman -Sy --noconfirm
|
||||||
|
--needed`, `dnf install -y`) for the minimal plan packages (`git`,
|
||||||
|
`ca-certificates`, `curl`, `tar`, `gzip`). No compiler is ever planned
|
||||||
|
or installed.
|
||||||
|
- **Toolchain.** `toolchainScript` downloads the pinned Go tarball into a
|
||||||
|
`mktemp` temp dir, verifies it with `sha256sum -c -` *before*
|
||||||
|
extraction, extracts into a staging dir on the same filesystem as
|
||||||
|
`ToolchainDir` (which must end in `/go`, default `/usr/local/go`),
|
||||||
|
verifies the staged toolchain reports the target version, and only then
|
||||||
|
swaps it into place. The prior toolchain is moved to a sibling
|
||||||
|
`.go-backup` and is restored if the swap fails, so a failed
|
||||||
|
download/verify/extract/swap always leaves the prior Go untouched. A
|
||||||
|
present toolchain that already reports the target version is reused, so
|
||||||
|
reruns do not re-download. Temp, staging, and backup directories are
|
||||||
|
removed on success and failure.
|
||||||
|
- **Clone/update.** `cloneUpdateScript` clones the repository when
|
||||||
|
`BuildDir` has no `.git` and otherwise fetches with `--prune`, so a
|
||||||
|
rerun updates in place. An existing checkout's `remote.origin.url` must
|
||||||
|
exactly match the configured repository before anything is fetched or
|
||||||
|
built, so the installer can never fetch or build an unconfigured
|
||||||
|
repository. The clone/fetch retries up to three times (2s apart)
|
||||||
|
because real repositories can be transiently unreachable (DNS, TLS, or
|
||||||
|
proxy hiccups); three bounded attempts keep a momentary outage from
|
||||||
|
failing a full source install.
|
||||||
|
- **Resolved branch.** The installer resolves the remote default branch
|
||||||
|
via `git remote set-head origin --auto` +
|
||||||
|
`git symbolic-ref --short refs/remotes/origin/HEAD`. When no branch is
|
||||||
|
pinned it builds the remote default (the public repo currently
|
||||||
|
publishes `master`); a pinned branch must exist remotely or the install
|
||||||
|
fails before the build. The resolved branch and the `git rev-parse
|
||||||
|
HEAD` commit (validated as 40 lowercase hex) are returned by
|
||||||
|
`SourceInstall`.
|
||||||
|
- **Staging build.** `buildScript` builds with `CGO_ENABLED=0`,
|
||||||
|
`-trimpath`, the repository's own `-ldflags` shape (version `dev`,
|
||||||
|
resolved commit short form, UTC build date), and both `GOCACHE` and
|
||||||
|
`GOMODCACHE` inside the build dir (so reruns reuse them), plus an
|
||||||
|
optional `GOPROXY`. The binary is built to a sibling `<stage>.new`,
|
||||||
|
verified with `<stage>.new --version`, and only then atomically swapped
|
||||||
|
over `<BuildDir>/rsmon-worker` (or `StageBinary`), so a failed build
|
||||||
|
never replaces the previous staging binary. It is not written to
|
||||||
|
`/usr/local/bin`.
|
||||||
|
- **Commit record.** The `rsmon-worker.commit` record (a
|
||||||
|
`branch=...` / `commit=...` format in the build dir) is written only
|
||||||
|
*after* a successful build, so the record and the staged binary always
|
||||||
|
correspond to the same commit.
|
||||||
|
|
||||||
|
Security properties of work package 3:
|
||||||
|
|
||||||
|
- Every interpolated value (repository, branch, build dir, URLs, SHA-256,
|
||||||
|
package names, paths) is single-quoted; repository, branch, commit, Go
|
||||||
|
version, and Go architecture values are additionally validated with
|
||||||
|
strict patterns. No worker token or control-plane credential is ever
|
||||||
|
sent: the install stages a binary and touches no service configuration.
|
||||||
|
- The repository must be an `https://` URL without userinfo, so source
|
||||||
|
credentials cannot reach the remote clone command or the clone's
|
||||||
|
config.
|
||||||
|
- Sudo passwords are delivered over the session's stdin only, never in a
|
||||||
|
command string (the same `sudoWrap` path the `deploy` command uses).
|
||||||
|
Direct `--password`/`--sudo-password`/`--key-passphrase` flags remain
|
||||||
|
available but expose the value through the process list and shell
|
||||||
|
history; the CLI docs strongly prefer the `-file` variants. The source
|
||||||
|
installer sends no worker token at all.
|
||||||
|
- Every step script fails closed: `set -eu` (or an explicit retry that
|
||||||
|
exits non-zero) is used, so a failed checkout or fetch can never be
|
||||||
|
masked by a stale subsequent command. The checkout step refuses before
|
||||||
|
the destructive `checkout -B` when the tracked working tree is dirty
|
||||||
|
(`git diff --quiet` / `--cached --quiet`), because `checkout -B` would
|
||||||
|
silently discard local changes; a dirty-tree rerun fails at checkout and
|
||||||
|
leaves the previous staging binary and commit record untouched.
|
||||||
|
- Remote errors are bounded: each step returns a step-labelled error,
|
||||||
|
stderr and captured stdout are size-bounded in `runRemoteOutput`, and
|
||||||
|
each remote command is capped by `--session-timeout` (default 30m).
|
||||||
|
- Toolchain temp, staging, and backup directories are removed on success
|
||||||
|
and failure, and the acceptance test asserts no `/tmp/rsmon-toolchain-*`
|
||||||
|
or `/usr/local/.go-staging-*`/`.go-backup` leaks after the rerun.
|
||||||
|
- Failed builds and failed checkouts leave the previous staging binary
|
||||||
|
untouched (the binary is only overwritten by an atomically-swapped
|
||||||
|
successful build, and a checkout failure aborts before the build).
|
||||||
|
|
||||||
|
## Work Package 4: Atomic Activation And Rollback
|
||||||
|
|
||||||
|
Work package 4 is the second half of `installer.SourceInstall`, driven by
|
||||||
|
`Activation` (`internal/installer/sourceactivate.go`). It runs over the
|
||||||
|
same SSH executor, privilege path, and bounded-error machinery as the
|
||||||
|
staging steps and installs the staged build atomically:
|
||||||
|
|
||||||
|
- **Layout.** The classic installer's on-disk layout is reused
|
||||||
|
(`resolvePaths`): binary `/usr/local/bin/rsmon-worker[-name]`, config
|
||||||
|
`/etc/rsmon-worker[-name]/worker.env` (mode 0600), data
|
||||||
|
`/var/lib/rsmon-worker[-name]`, and a service definition for the
|
||||||
|
detected init (systemd unit `/etc/systemd/system/rsmon-worker.service`
|
||||||
|
mode 0644, or an OpenRC script `/etc/init.d/rsmon-worker` mode 0755).
|
||||||
|
A named instance (`--name`) mirrors the classic multi-instance
|
||||||
|
convention with `-<name>` suffixes and its own unit and port.
|
||||||
|
- **Environment.** The env file is rendered by the classic installer's
|
||||||
|
`resolveInstallEnv` + `renderEnvFile` (so `PUBLIC_URL` canonicalization,
|
||||||
|
basic-auth XOR, required `RSMON_URL`/`RSMON_TOKEN`, and systemd-safe
|
||||||
|
value validation are identical to `install`). The env file is read
|
||||||
|
*exactly once* and validated from the in-memory bytes
|
||||||
|
(`parseEnvironmentContent`), and the rendered env is cached during
|
||||||
|
option normalization and reused at activation time, so a hostile local
|
||||||
|
writer cannot swap the file between the preflight and the remote
|
||||||
|
install (env-file TOCTOU). The rendered bytes are uploaded into a
|
||||||
|
server-created 0700 temp dir (`mktemp -d /tmp/rsmon-worker-act.XXXXXX`
|
||||||
|
under the SSH user) rather than a predictable `/tmp` path, eliminating
|
||||||
|
the symlink/TOCTOU attack surface on the upload while the token still
|
||||||
|
travels only as base64 over the session stdin and, later, only inside
|
||||||
|
the mode-0600 env file. The env is then installed with mode 0600 in the
|
||||||
|
config dir (mode 0750).
|
||||||
|
- **Atomic install.** The staged binary is validated (`<stage>
|
||||||
|
--version`) before any state is touched, then installed with a
|
||||||
|
same-directory temp file + rename. The env and unit files are installed
|
||||||
|
the same way. The data dir (and `webapp/` subdir) is created with mode
|
||||||
|
0755.
|
||||||
|
- **Supervisor.** `restart_svc` drives start/restart through whichever
|
||||||
|
init system is *actually running*: systemd (`systemctl`), OpenRC
|
||||||
|
(`rc-service`), or the embedded supervisor fallback when no init is
|
||||||
|
running (containers/chroots, the "no-service gate"). The health loop
|
||||||
|
verifies the service is active *through the supervisor* — `systemctl
|
||||||
|
is-active` for systemd, `rc-service status` for OpenRC, and a
|
||||||
|
zombie-aware pid check for the embedded supervisor — never through a
|
||||||
|
pid file on the init-managed paths. The embedded supervisor keeps the
|
||||||
|
worker as a single background process with a pid file under the data
|
||||||
|
dir, stops the previous instance before starting, refuses to kill a pid
|
||||||
|
whose executable is not the configured worker (`/proc/<pid>/exe`
|
||||||
|
checked before every `kill`), and answers `/healthz` via the worker's
|
||||||
|
own `liveness` subcommand with a clean `env -i` environment built from
|
||||||
|
the env file (so a rolled-back env can never leak a stale variable into
|
||||||
|
the restored process). The OpenRC init script exports the env-file
|
||||||
|
variables (`set -a` before sourcing) so the worker inherits them when
|
||||||
|
OpenRC starts it.
|
||||||
|
- **Rollback.** The prior binary, env, and unit (and the unit's enable
|
||||||
|
state) are snapshotted into the data dir before any mutation, and an
|
||||||
|
`EXIT`/`HUP`/`INT`/`TERM` trap restores them (preserving the prior
|
||||||
|
binary/env/unit metadata via `cp -p`) and restarts the prior service if
|
||||||
|
any later step fails or a signal interrupts the run. A corrupt staged
|
||||||
|
binary fails the pre-validation *before* the trap is armed, so the
|
||||||
|
prior install is never touched. A fresh-install failure disables the
|
||||||
|
newly-enabled unit (`systemctl disable` / `rc-update del`) and removes
|
||||||
|
it; a rerun failure restores the prior unit and re-applies its prior
|
||||||
|
enable state. Rollback also removes the backup dir, the `.new` temp
|
||||||
|
files, the uploaded env/unit temps, and the activation lock.
|
||||||
|
- **Lock, recovery, interruption.** A `mkdir`-based activation lock
|
||||||
|
(`.rsmon-activate.lock`, broken automatically when its recorded pid is
|
||||||
|
dead) prevents concurrent activations from racing the deployed-state
|
||||||
|
mutation. The snapshot is marked; a run killed mid-flight (SSH drop,
|
||||||
|
SIGKILL) leaves that marker, and the next activation restores the
|
||||||
|
leftover snapshot before proceeding, so the host never stays
|
||||||
|
half-activated.
|
||||||
|
- **Idempotency.** A rerun reuses the existing env (rewriting it
|
||||||
|
deterministically from the same knobs), swaps the binary atomically,
|
||||||
|
restarts exactly one worker, and leaves no `.rsmon-backup`, `.new`,
|
||||||
|
`/tmp/rsmon-worker-act.*`, or lock leftovers. `--no-start` installs the
|
||||||
|
full layout without starting anything and reruns stay idle.
|
||||||
|
|
||||||
|
The `source-install` CLI activates by default; pass `--no-activate` for
|
||||||
|
the staging-only behavior of work package 3. The worker token is required
|
||||||
|
for activation and is supplied with `--token`/`--token-file` or
|
||||||
|
`--env-file` (prefer the file options; direct flags expose the value
|
||||||
|
through the process list).
|
||||||
|
|
||||||
|
## Work Package 5: Alpine/Ubuntu/Arch E2E In CI
|
||||||
|
|
||||||
|
Work package 5 runs the full Docker/OpenSSH E2E matrix on pushes to
|
||||||
|
`master` and on manual `workflow_dispatch` runs. It is a separate Gitea
|
||||||
|
Actions workflow (`.github/workflows/test-ssh.yml`) so the ordinary unit
|
||||||
|
CI run stays Docker-free; the job executes `make test-ssh` through
|
||||||
|
`scripts/ci/test-ssh.sh`.
|
||||||
|
|
||||||
|
CI behavior:
|
||||||
|
|
||||||
|
- **Trust boundary and triggers.** The workflow runs *only* on pushes to
|
||||||
|
`master` (the default branch) and on manual `workflow_dispatch`. It
|
||||||
|
deliberately does **not** trigger on `pull_request`: the distro fixtures
|
||||||
|
execute the checked-out repository code inside privileged Docker
|
||||||
|
containers, so an untrusted PR must never reach the runner's Docker
|
||||||
|
surface automatically. Gitea's read-only token clamp for fork PRs does
|
||||||
|
not limit what containers can do on the runner host, so PR coverage is
|
||||||
|
left to the Docker-free unit CI (`ci.yml`) and to manual dispatch after
|
||||||
|
a human reviews the change.
|
||||||
|
- **Concurrency is scoped per ref.** `concurrency.group:
|
||||||
|
test-ssh-${{ gitea.ref }}` gives master pushes their own group (a newer
|
||||||
|
master push cancels a superseded in-flight master run instead of
|
||||||
|
stacking) and gives a manual dispatch on another branch its own group so
|
||||||
|
it never cancels the master run. Gitea Actions evaluates the expression,
|
||||||
|
and `gitea.ref` is the same documented context the repo's `docker.yml`
|
||||||
|
already uses.
|
||||||
|
- **Least privilege.** The workflow declares `permissions: contents: read`
|
||||||
|
(supported by Gitea Actions as the `GITEA_TOKEN` scope for
|
||||||
|
code/releases), so the job's token can only read the repository; the
|
||||||
|
workflow never writes, pushes, or publishes.
|
||||||
|
- **Action revisions.** The two actions this workflow uses are pinned to
|
||||||
|
immutable full commit SHAs (not moving tags): `actions/checkout@v4` ->
|
||||||
|
`11d5960a326750d5838078e36cf38b85af677262` and `actions/setup-go@v5` ->
|
||||||
|
`40f1582b2485089dde7abd97c1529aa768e1baff` (verified 2026-08-13 against
|
||||||
|
the GitHub API that each tag points to a commit object). The repo-wide
|
||||||
|
convention still leaves `ci.yml` and `docker.yml` on moving tags
|
||||||
|
(`@v4`, `@v5`, `@v3`, `@v6`); that is an accepted, documented risk: a
|
||||||
|
tag move can change behavior without a workflow diff. New workflows
|
||||||
|
should pin SHAs like this one; migrating the existing workflows is a
|
||||||
|
separate change.
|
||||||
|
- **Runner requirement.** The harness dials fixture SSH ports published on
|
||||||
|
the Docker daemon's `127.0.0.1`, so the job must share the daemon's
|
||||||
|
loopback (a host-mode runner or a job container with host networking).
|
||||||
|
`scripts/ci/test-ssh.sh` probes this with a tiny `docker run -p
|
||||||
|
127.0.0.1::22` round trip *before* the matrix and fails fast with a
|
||||||
|
clear, actionable message (including diagnostics and fix options)
|
||||||
|
instead of after a 60m go-test timeout.
|
||||||
|
- **Bounding and timeout.** The workflow sets `timeout-minutes: 90` and
|
||||||
|
the existing `make test-ssh` go-test `-timeout 60m` stays in place, so
|
||||||
|
the whole job is hard-bounded even under slow networks or downloads.
|
||||||
|
- **Cleanup.** The harness already tears down every container, network,
|
||||||
|
per-instance fixture image tag, and temp dir on success and failure.
|
||||||
|
`scripts/ci/test-ssh.sh` additionally traps `EXIT` to remove any
|
||||||
|
leftover `rsmon-worker-test-*` resource (container, network, or the
|
||||||
|
per-instance `rsmon-worker-test/<fixture>-<suffix>:local` image tag), and
|
||||||
|
the workflow adds an `if: always()` step that does the same even when the
|
||||||
|
script itself is killed. All filters are anchored to the harness's own
|
||||||
|
prefix and image repository (`name=^rsmon-worker-test-`,
|
||||||
|
`reference=rsmon-worker-test/*`), so cleanup never touches a shared base
|
||||||
|
image (`reg.rsxx.ru/library/alpine:3`, `ubuntu:24.04`, `archlinux:latest`)
|
||||||
|
or an unrelated resource.
|
||||||
|
- **Mirrors and overrides.** The Alpine fixture already uses the
|
||||||
|
`reg.rsxx.ru/library/alpine:3` mirror. Ubuntu and Arch have no mirror yet
|
||||||
|
and default to Docker Hub. The workflow wires the
|
||||||
|
`RSMON_TEST_IMAGE_ALPINE`, `RSMON_TEST_IMAGE_UBUNTU`, and
|
||||||
|
`RSMON_TEST_IMAGE_ARCH` repository variables (empty by default) so an
|
||||||
|
operator can pin a mirror or a specific distro snapshot per fixture, and
|
||||||
|
the `RSMON_TEST_DOCKER_DNS` repository variable (comma-separated
|
||||||
|
nameservers, applied as `docker run --dns ...`) to pin a resolver for
|
||||||
|
flaky CI DNS. External network (go.dev toolchain download, the public
|
||||||
|
rocketgit.ru source clone, distro package repos) is fetched live by
|
||||||
|
design.
|
||||||
|
- **Artifacts and secrets.** The workflow uploads no artifacts: the go-test
|
||||||
|
log stays in the runner's job log and nothing private (test key,
|
||||||
|
`known_hosts`, env files) is retained on the runner or published. The
|
||||||
|
fixture containers receive only the fixed `e2e-activation-test-token`
|
||||||
|
and the bundled test-only key, never a real worker token.
|
||||||
|
- **Caching.** `actions/setup-go` caches the Go module/build cache used to
|
||||||
|
compile the harness test binary (public dependencies only). Fixture and
|
||||||
|
toolchain downloads are not cached because they run inside disposable
|
||||||
|
distro containers; reruns rebuild them cleanly.
|
||||||
|
|
||||||
|
Work package 5 does not change how the source installer behaves. It only
|
||||||
|
adds a CI surface for the existing acceptance tests.
|
||||||
|
|
||||||
|
## Docker OpenSSH Test Harness
|
||||||
|
|
||||||
|
Adapt the real-network pattern from `/data/_swap/sshkeymanager`: start an
|
||||||
|
OpenSSH container, wait for SSH readiness, connect with the Go installer, and
|
||||||
|
tear the environment down reliably. Do not mock SSH command execution in the
|
||||||
|
acceptance test.
|
||||||
|
|
||||||
|
Provide images/fixtures for:
|
||||||
|
|
||||||
|
- Ubuntu with apt and systemd-compatible service testing where practical;
|
||||||
|
- Alpine with apk/OpenRC or a clearly separated no-service build/install gate;
|
||||||
|
- Arch with pacman and its service behavior.
|
||||||
|
|
||||||
|
Each clean target begins without Go or the worker source. The test asserts
|
||||||
|
package installation, verified Go version, clone branch/resolved commit, build,
|
||||||
|
atomic config permissions, running service where supported, and HTTP liveness.
|
||||||
|
|
||||||
|
### Implementation (`internal/installer/harness`)
|
||||||
|
|
||||||
|
The harness starts one container per distro fixture
|
||||||
|
(`testdata/fixtures/{alpine,ubuntu,arch}/Dockerfile`), waits for a real
|
||||||
|
TCP + SSH handshake, captures the server host key into a temp
|
||||||
|
`known_hosts` file, and connects with the `golang.org/x/crypto/ssh`
|
||||||
|
library and known_hosts verification semantics the installer's `deploy`
|
||||||
|
uses. The harness owns its connection code rather than calling into the
|
||||||
|
installer package, so the tests stay independent; only the library and
|
||||||
|
the verification semantics are shared.
|
||||||
|
|
||||||
|
Host-key handling is trust-on-first-use (TOFU): the fresh temp
|
||||||
|
`known_hosts` file accepts the first key the server presents. What the
|
||||||
|
harness proves is that a known_hosts entry carrying a *different* key is
|
||||||
|
rejected before any command runs (the `TestHarnessHostKeyMismatch` test),
|
||||||
|
not that a fingerprint is pinned.
|
||||||
|
|
||||||
|
The fixtures install over the public internet, so environments with flaky
|
||||||
|
local resolvers can pin a reliable upstream via the comma-separated
|
||||||
|
`RSMON_TEST_DOCKER_DNS` variable (applied as `docker run --dns ...`); it
|
||||||
|
is empty by default, keeping Docker's embedded DNS.
|
||||||
|
|
||||||
|
Teardown (`docker rm -f` + `docker network rm` + per-instance
|
||||||
|
`docker image rm` + temp-dir removal) is idempotent, runs on every
|
||||||
|
`Start` error path, and is verified by a dedicated test. Each harness
|
||||||
|
instance builds its own uniquely-tagged fixture image
|
||||||
|
(`rsmon-worker-test/<fixture>-<suffix>:local`), so removing one never
|
||||||
|
deletes a shared base image or another instance's image.
|
||||||
|
|
||||||
|
Base images, mirror-first:
|
||||||
|
|
||||||
|
| Fixture | Default image | Note |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| alpine | `reg.rsxx.ru/library/alpine:3` | reg.rsxx.ru mirror exists |
|
||||||
|
| ubuntu | `ubuntu:24.04` | no mirror yet; override `RSMON_TEST_IMAGE_UBUNTU` |
|
||||||
|
| arch | `archlinux:latest` | no mirror yet; override `RSMON_TEST_IMAGE_ARCH` |
|
||||||
|
|
||||||
|
Any `RSMON_TEST_IMAGE_<NAME>` environment variable overrides the fixture
|
||||||
|
image, so a mirror or local cache can be used when available.
|
||||||
|
|
||||||
|
> **Mutable test images.** The fixture base tags above are deliberately
|
||||||
|
> mutable (a major-tag mirror ref and Docker Hub rolling tags) so the
|
||||||
|
> fixtures track current distro releases. Fixture builds are therefore
|
||||||
|
> not byte-reproducible; the worker's *source-install* production output
|
||||||
|
> pins immutable artifacts (a Go toolchain SHA-256, a branch's resolved
|
||||||
|
> commit) and this harness's image override is the escape hatch for
|
||||||
|
> reproducing a specific distro snapshot.
|
||||||
|
|
||||||
|
The fixtures authenticate with the bundled test key
|
||||||
|
(`testdata/keys/rsmon_test_ed25519`); password auth is disabled and
|
||||||
|
`PermitRootLogin` is `prohibit-password`. `openrc` is installed in the
|
||||||
|
Alpine fixture so init detection has a stable marker; Ubuntu and Arch
|
||||||
|
carry systemd markers.
|
||||||
|
|
||||||
|
> **Test-only key.** The bundled keypair is strictly a test fixture: it
|
||||||
|
> grants root SSH access only to the disposable containers that bake its
|
||||||
|
> public key. It must never be used for real hosts, added to production
|
||||||
|
> images, or treated as a credential outside the harness.
|
||||||
|
|
||||||
|
### Opt-in integration test controls
|
||||||
|
|
||||||
|
Ordinary unit runs never pull or start Docker. The Docker/OpenSSH tests
|
||||||
|
are gated behind the `RSMON_TEST_DOCKER` environment variable:
|
||||||
|
|
||||||
|
- `make test` (default CI unit run) pins `RSMON_TEST_DOCKER=0` and skips
|
||||||
|
every harness Docker test, even when the flag is exported in the
|
||||||
|
developer's environment.
|
||||||
|
- `make test-ssh` sets `RSMON_TEST_DOCKER=1` and runs the full fixture
|
||||||
|
matrix (`TestHarnessFixtures` for alpine/ubuntu/arch, host-key
|
||||||
|
mismatch, stable host key, failed-start cleanup, and teardown
|
||||||
|
assertions).
|
||||||
|
- The harness `Start` itself refuses to run without the opt-in flag.
|
||||||
|
|
||||||
|
Run them locally with:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
make test-ssh
|
||||||
|
```
|
||||||
|
|
||||||
|
or, equivalently:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
RSMON_TEST_DOCKER=1 go test -v -count=1 -timeout 30m ./internal/installer/harness
|
||||||
|
```
|
||||||
|
|
||||||
|
## Idempotency And Security
|
||||||
|
|
||||||
|
- A second run updates/fetches safely: the toolchain is reused when the version
|
||||||
|
matches, the clone's origin is verified against the configured repository and
|
||||||
|
then fetched in place, the resolved branch/commit record is rewritten after
|
||||||
|
the new build succeeds, and the staging build atomically swaps over the
|
||||||
|
previous staging binary. Activation (work package 4) makes the rerun
|
||||||
|
idempotent at the service level too: exactly one worker process exists, the
|
||||||
|
env file is rewritten deterministically from the same knobs, and no
|
||||||
|
`.rsmon-backup`, `.new`, `/tmp/rsmon-worker-act.*`, or lock files leak.
|
||||||
|
- Wrong host fingerprints fail before remote mutation. The harness's fresh
|
||||||
|
known_hosts file is trust-on-first-use; its dedicated mismatch test dials
|
||||||
|
against a known_hosts entry carrying a different server key and proves the
|
||||||
|
dial fails before any command runs.
|
||||||
|
- Tokens/passwords should come from files: `--key-passphrase-file`,
|
||||||
|
`--password-file`, `--sudo-password-file`, and the activation
|
||||||
|
`--token-file`/`--env-file` keep secrets out of argv and shell history,
|
||||||
|
while the equivalent direct flags expose them through the process list. The
|
||||||
|
worker token is written only to the mode-0600 env file, never echoed into a
|
||||||
|
command or log, and sudo passwords travel only over the session's stdin. The
|
||||||
|
env and unit uploads land in a server-created 0700 `mktemp -d` directory (not
|
||||||
|
a predictable `/tmp` path), so no local user can plant a symlink at the
|
||||||
|
upload target (TOCTOU), and the env file itself is read/rendered exactly once
|
||||||
|
so a local writer cannot swap it between validation and upload.
|
||||||
|
- Remote temporary files are removed on success and failure, including the
|
||||||
|
activation backup dir, the uploaded env/unit temps, and the activation lock.
|
||||||
|
The snapshot is written under a marker so a run killed mid-flight is
|
||||||
|
recovered (restored) by the next activation instead of leaving the host
|
||||||
|
half-activated.
|
||||||
|
- Failed builds, failed checkouts, and failed toolchain swaps do not replace a
|
||||||
|
working binary or service definition. Failed activations, failed starts, and
|
||||||
|
failed `/healthz` verifications restore the prior binary, env, and service
|
||||||
|
definition (preserving their metadata), re-apply the prior unit enable state
|
||||||
|
(or disable a freshly-enabled unit on a fresh failure), and bring the prior
|
||||||
|
worker back up (work package 4 rollback). The supervisor only ever kills a
|
||||||
|
process whose `/proc/<pid>/exe` matches the configured worker binary, so a
|
||||||
|
stale or recycled pid file can never kill an unrelated process.
|
||||||
|
- Package-manager and download failures return bounded actionable errors.
|
||||||
|
- The installer verifies Go tarball checksum before extraction.
|
||||||
|
|
||||||
|
## Implementation Work Packages
|
||||||
|
|
||||||
|
- [x] 1. Add reusable Docker/OpenSSH harness and distro fixtures.
|
||||||
|
- [x] 2. Add pure distro/toolchain/source-install script planning and unit tests
|
||||||
|
(detection + planning foundation; remote execution is work package 3).
|
||||||
|
- [x] 3. Execute source installation through the existing SSH transport
|
||||||
|
(prerequisites, verified Go toolchain, clone/update, resolved branch and
|
||||||
|
commit record, and a build to a staging path; no service activation).
|
||||||
|
- [x] 4. Add atomic build/install, idempotency, and failure rollback (binary,
|
||||||
|
env, data dir, and init service definition installed atomically; process
|
||||||
|
and `/healthz` verified; activation/start/health failures restore the
|
||||||
|
prior install; reruns keep exactly one service).
|
||||||
|
- [x] 5. Add Alpine, Ubuntu, and Arch network E2E tests to CI.
|
||||||
|
- [ ] 6. Add CentOS-family support.
|
||||||
|
- [ ] 7. Plan native Windows service and macOS launchd installers separately.
|
||||||
|
|
||||||
|
## Acceptance Gates
|
||||||
|
|
||||||
|
- [x] All three initial Linux images install from a clean state through OpenSSH:
|
||||||
|
package install, verified Go 1.26, clone/update, resolved commit, staging
|
||||||
|
build, and (work package 4) atomic activation with a running verified
|
||||||
|
worker.
|
||||||
|
- [x] The built worker reports the expected version/commit and serves `/healthz`
|
||||||
|
(verified by the E2E fixture tests and by the activation health gate).
|
||||||
|
- [x] Re-running the installer succeeds without duplicate services or leaked
|
||||||
|
files (exactly one worker process and no backup/temp leftovers asserted by
|
||||||
|
the rerun test on all three fixtures).
|
||||||
|
- [x] Host-key, checksum, clone, build, and service-start failure tests preserve
|
||||||
|
the previous installation (build/checkout, activation, start, and health
|
||||||
|
failure reruns on the Alpine fixture all restore the prior binary, env,
|
||||||
|
and running service).
|
||||||
|
- [x] CI uses approved registry mirrors and cleans every test container/network.
|
||||||
|
|
||||||
|
## Verified Test Evidence (work packages 1-4)
|
||||||
|
|
||||||
|
Recorded 2026-08-12 from `make test-ssh` (Docker Engine 29.7.1):
|
||||||
|
|
||||||
|
- Alpine `3.24.1` (mirror `reg.rsxx.ru/library/alpine:3`): distro alpine,
|
||||||
|
pkg apk, init openrc.
|
||||||
|
- Ubuntu `24.04` (`VERSION_ID=24.04`): distro ubuntu, pkg apt, init systemd.
|
||||||
|
- Arch rolling image `archlinux:latest` (`VERSION_ID=20260809.0.570793`):
|
||||||
|
distro arch, pkg pacman, init systemd.
|
||||||
|
- Host-key mismatch, host-key stability, failed-start cleanup, and complete
|
||||||
|
teardown (container, network, fixture image tag, and temp dir gone) tests
|
||||||
|
pass; no test container, network, or image tag is left behind.
|
||||||
|
- `TestSourceInstallFixtures` runs the full work-package-3 flow on each
|
||||||
|
fixture from a clean state over the harness's real OpenSSH transport:
|
||||||
|
prerequisite install, verified Go 1.26 toolchain download/extraction, clone
|
||||||
|
of the public repository, resolution of its default branch, resolved-commit
|
||||||
|
record file, and a staging build that reports the resolved commit via
|
||||||
|
`--version`. A rerun succeeds, keeps the same branch, reuses the toolchain,
|
||||||
|
and leaves no `/tmp/rsmon-toolchain-*` temp dirs.
|
||||||
|
- `TestSourceInstallActivationFixtures` runs the full work-package-4 flow on
|
||||||
|
each fixture from a clean state: after the staging build the installer
|
||||||
|
atomically installs the binary (`/usr/local/bin/rsmon-worker`), env
|
||||||
|
(`/etc/rsmon-worker/worker.env`, mode 0600; config dir 0750), data dir
|
||||||
|
(`/var/lib/rsmon-worker`), and the detected init's service definition
|
||||||
|
(systemd unit on Ubuntu/Arch, OpenRC script on Alpine), then starts the
|
||||||
|
worker via the embedded supervisor (no init runs inside the fixtures) and
|
||||||
|
verifies exactly one worker process and a live `/healthz`. A rerun succeeds,
|
||||||
|
keeps a single process, rewrites the env deterministically, and leaves no
|
||||||
|
`.rsmon-backup`, `.new`, `/tmp/rsmon-worker-act.*`, or lock leftovers.
|
||||||
|
- `TestSourceInstallActivationFailureRollback` (Alpine) forces four rerun
|
||||||
|
failures and proves each restores the prior install byte-for-byte and
|
||||||
|
running: a dirty checkout (build failure), a sabotaged atomic swap
|
||||||
|
(activation failure), `WORKER_CLUSTER_ENABLED=true` without credentials (the
|
||||||
|
new process exits at boot; start failure), and `WORKER_HOST=255.255.255.255`
|
||||||
|
(the new process runs but `/healthz` is unreachable; health failure). In
|
||||||
|
every case the installed binary SHA-256, env, service definition, single
|
||||||
|
process, and liveness match the pre-failure state, and no backup, lock, or
|
||||||
|
temp files leak.
|
||||||
|
- `TestSourceInstallActivationNoStart` installs the full layout without
|
||||||
|
starting anything, and reruns stay idle.
|
||||||
|
- The targeted shell fixture tests (`TestActivateShell*`) execute the real
|
||||||
|
activation script against stub `systemctl`/`rc-service`/`rc-update`
|
||||||
|
implementations and cover the init-managed paths the Docker fixtures cannot
|
||||||
|
reach: systemd unit install/enable with a `systemctl is-active`-based health
|
||||||
|
loop (and no pid file), OpenRC install/enable with an `rc-service`-based
|
||||||
|
health loop, fresh-failure rollback disabling a newly-enabled unit, rerun
|
||||||
|
rollback restoring the prior unit and its enable state, stale-lock breaking,
|
||||||
|
and interrupted-run recovery from a leftover backup marker.
|
||||||
|
- The unit suite covers the per-step remote scripts, the activation script
|
||||||
|
(backup + recovery marker, activation lock, atomic install, rollback trap,
|
||||||
|
supervisor, pid/zombie handling, pid-belongs-to-binary checks, enable-state
|
||||||
|
restoration, secrets absent), OpenRC unit rendering (including env
|
||||||
|
export), option validation, branch/commit parsing, sudo wrapping (password
|
||||||
|
never in the command), and an in-process real-SSH orchestration flow with
|
||||||
|
failure paths for missing pinned branches, build failures, detection
|
||||||
|
failures, and activation/start/health failures.
|
||||||
|
- **Residual limitation (stated honestly):** live systemd and OpenRC cannot
|
||||||
|
run inside the Docker/OpenSSH harness (the container PID 1 is sshd), so the
|
||||||
|
init-managed supervisor paths are verified by the stub-based shell fixture
|
||||||
|
tests above rather than against a real init. The stub tools simulate unit
|
||||||
|
state and command flow, not real systemd/OpenRC unit semantics; a real
|
||||||
|
init-system smoke test on a booted host remains a follow-up.
|
||||||
|
|
||||||
|
Work package 5 (CI) evidence, recorded 2026-08-13:
|
||||||
|
|
||||||
|
- `.github/workflows/test-ssh.yml` runs `scripts/ci/test-ssh.sh` on pushes
|
||||||
|
to `master` and on manual `workflow_dispatch` only (never on
|
||||||
|
`pull_request`, because the fixtures execute the checked-out code inside
|
||||||
|
privileged Docker). Concurrency is scoped per ref
|
||||||
|
(`test-ssh-${{ gitea.ref }}`), the job token is `contents: read`, and
|
||||||
|
the script's loopback port-publishing probe passes on a host Docker
|
||||||
|
daemon while its `EXIT` cleanup leaves zero leftover containers,
|
||||||
|
networks, or fixture image tags.
|
||||||
|
- A full local `make test-ssh` run (the exact command the CI job executes)
|
||||||
|
passes the complete harness suite: `TestHarnessFixtures` (alpine/ubuntu/
|
||||||
|
arch), host-key mismatch and stability, teardown, `TestSourceInstallFixtures`
|
||||||
|
(work package 3 staging on all three distros), `TestSourceInstallActivationFixtures`
|
||||||
|
(work package 4 activation on all three), activation failure rollback,
|
||||||
|
`--no-start`, and the dirty-checkout preservation test. Pinned
|
||||||
|
`RSMON_TEST_DOCKER_DNS=1.1.1.1` was used because the local network's
|
||||||
|
default resolvers intermittently time out on `rocketgit.ru`.
|
||||||
|
- `TestSourceInstallDirtyCheckoutPreservesStaging` and the rollback test's
|
||||||
|
build-failure step now dirty the tracked tree deterministically (appending
|
||||||
|
a marker line to `Makefile`) instead of `git checkout master~1 -- Makefile`,
|
||||||
|
which depended on the last two commits differing and silently stopped
|
||||||
|
dirtying the tree once a commit did not touch that file.
|
||||||
@@ -39,10 +39,10 @@ Conversion date: 2026-07-13.
|
|||||||
| --- | --- | --- |
|
| --- | --- | --- |
|
||||||
| `docs/inventory-management.md` | `inventory.md` | Control-plane projection retained as dependency; worker owns collection only |
|
| `docs/inventory-management.md` | `inventory.md` | Control-plane projection retained as dependency; worker owns collection only |
|
||||||
| `docs/plans/inventory-management.md` | `inventory.md` | deploymentd and worker reports defined as source-aware complementary producers |
|
| `docs/plans/inventory-management.md` | `inventory.md` | deploymentd and worker reports defined as source-aware complementary producers |
|
||||||
| `docs/parity/rstuff-inventory.md` | `inventory.md` | rstuff enums/stream remain control-plane contracts, not worker protocol |
|
| `docs/parity/rstuff-inventory.md` | `inventory.md` | Historical only; RSMon owns inventory and RSLogin supplies project/access input |
|
||||||
| `docs/parity/deploymentd.md` | `inventory.md` | deploymentd parity becomes fixture normalization and comparison gate |
|
| `docs/parity/deploymentd.md` | `inventory.md` | deploymentd parity becomes fixture normalization and comparison gate |
|
||||||
| `docs/deploymentd-replay-rollout.md` | `inventory.md` | HTTP nonce rollout remains deploymentd-specific; worker uses authenticated typed reports |
|
| `docs/deploymentd-replay-rollout.md` | `inventory.md` | HTTP nonce rollout remains deploymentd-specific; worker uses authenticated typed reports |
|
||||||
| `docs/integrations/rstuff-inventory-stream.md` | `inventory.md` | Valkey stream remains between control plane and rstuff, never worker transport |
|
| `docs/integrations/rstuff-inventory-stream.md` | `inventory.md` | Superseded; no rstuff service is planned and Valkey is not the target inventory authority |
|
||||||
| `docs/plans/servers-and-hardware-metrics.md` | `web-console-and-observability.md`, `inventory.md` | Implemented server metric report retained; local hardware extensions staged |
|
| `docs/plans/servers-and-hardware-metrics.md` | `web-console-and-observability.md`, `inventory.md` | Implemented server metric report retained; local hardware extensions staged |
|
||||||
| `docs/server-observability.md` | `web-console-and-observability.md`, `control-plane-protocol.md` | Current wire fields, ownership validation, and TSDB boundary retained |
|
| `docs/server-observability.md` | `web-console-and-observability.md`, `control-plane-protocol.md` | Current wire fields, ownership validation, and TSDB boundary retained |
|
||||||
| `docs/parity/capture.md` | `web-console-and-observability.md`, `implementation-roadmap.md` | Worker-host coverage gaps converted into staged collector work |
|
| `docs/parity/capture.md` | `web-console-and-observability.md`, `implementation-roadmap.md` | Worker-host coverage gaps converted into staged collector work |
|
||||||
@@ -101,9 +101,10 @@ obsolete. Collector lifecycle remains a real implementation gap.
|
|||||||
### Inventory
|
### Inventory
|
||||||
|
|
||||||
deploymentd HTTP ingest is currently implemented on the control plane. Worker
|
deploymentd HTTP ingest is currently implemented on the control plane. Worker
|
||||||
inventory is local/partial and has no upstream report yet. Future worker and
|
inventory is local/partial and has no upstream report yet. RSMon is the target
|
||||||
deploymentd reports are complementary source-scoped inputs; neither may erase
|
inventory authority; RSLogin supplies trusted project and SSH-access input.
|
||||||
the other's observations.
|
Future worker and deploymentd reports are complementary source-scoped inputs;
|
||||||
|
neither may erase the other's observations.
|
||||||
|
|
||||||
### Private Workers And Public Work
|
### Private Workers And Public Work
|
||||||
|
|
||||||
|
|||||||
@@ -17,9 +17,11 @@ The implemented path is:
|
|||||||
|
|
||||||
Implemented check kinds are HTTP, SSL, SSH, FTP, DNS, WHOIS, BSSL, LLM,
|
Implemented check kinds are HTTP, SSL, SSH, FTP, DNS, WHOIS, BSSL, LLM,
|
||||||
LLM-HTTP, ping, TCP, and UDP. Control-plane selection uses capabilities. The
|
LLM-HTTP, ping, TCP, and UDP. Control-plane selection uses capabilities. The
|
||||||
local dispatcher supports only its explicit switch cases, but strict malformed
|
runner and local executor share one supported-kind registry. Websocket task
|
||||||
envelope and unsupported-kind result handling remains work package N1 rather
|
envelopes require exactly one payload branch, matching non-empty outer/inner
|
||||||
than a complete fail-closed protocol response.
|
job IDs, and a lease token. Unsupported checks return one terminal failed
|
||||||
|
result without execution. A safely attributable malformed single branch
|
||||||
|
returns a protocol failure; ambiguous and mismatched envelopes fail closed.
|
||||||
|
|
||||||
## Queue And Shutdown Rules
|
## Queue And Shutdown Rules
|
||||||
|
|
||||||
@@ -29,8 +31,9 @@ than a complete fail-closed protocol response.
|
|||||||
maximum.
|
maximum.
|
||||||
- A task panic is recovered at the task boundary and reported as a failed
|
- A task panic is recovered at the task boundary and reported as a failed
|
||||||
attempt; it must not kill the runner.
|
attempt; it must not kill the runner.
|
||||||
- SIGTERM stops accepting tasks, allows bounded in-flight completion, attempts
|
- SIGTERM stops accepting new tasks and waits for active dispatchers before the
|
||||||
final result delivery, and then exits.
|
execution pool closes. A bounded final-result drain across websocket shutdown
|
||||||
|
is not implemented yet.
|
||||||
- WebSocket reconnect does not re-run an in-flight or completed task.
|
- WebSocket reconnect does not re-run an in-flight or completed task.
|
||||||
- HTTP polling remains compatibility-only and must not become a second normal
|
- HTTP polling remains compatibility-only and must not become a second normal
|
||||||
scheduler.
|
scheduler.
|
||||||
@@ -83,26 +86,27 @@ unreachable. They use system contacts and credentials from config, local
|
|||||||
jitter, and local deduplication. They must not impersonate customer delivery or
|
jitter, and local deduplication. They must not impersonate customer delivery or
|
||||||
mutate control-plane message state.
|
mutate control-plane message state.
|
||||||
|
|
||||||
The local notifications page currently records selfcheck deliveries. Normal
|
The local notifications page records both selfcheck and delegated delivery
|
||||||
delegated delivery attempts should be added to the same bounded view after
|
attempts. Delegated rows contain job ID, method, status, duration, and time
|
||||||
redaction, with job ID, method, status, duration, and time only.
|
only. Recipient, body, credentials, hook URL, authorization, provider response,
|
||||||
|
and error text are never copied into the bounded ring. Invalid deadlines and
|
||||||
|
recovered executor panics produce static permanent results and one redacted
|
||||||
|
local row.
|
||||||
|
|
||||||
## Implementation Work Packages
|
## Implementation Work Packages
|
||||||
|
|
||||||
### N1: Runner Integration Coverage
|
### N1: Runner Integration Coverage (partial)
|
||||||
|
|
||||||
Add strict envelope/inner job-ID validation and structured unsupported-kind
|
Strict envelope/inner job-ID/lease validation and structured unsupported-kind
|
||||||
results. Add WebSocket integration tests for reconnect, task panic recovery,
|
results are implemented with runner-boundary tests. Live WebSocket reconnect
|
||||||
queue backpressure, result resend, stale lease behavior, malformed envelopes,
|
and result-resend tests are implemented. Stale-lease acknowledgement and
|
||||||
unsupported kinds, and graceful drain. Target
|
bounded graceful final-drain tests remain.
|
||||||
`internal/distworker/runner_protocol_test.go` and a local test WebSocket server.
|
|
||||||
|
|
||||||
### N2: Complete Local Delivery Audit
|
### N2: Complete Local Delivery Audit (implemented)
|
||||||
|
|
||||||
Record delegated notification outcomes in the bounded local notification ring.
|
Delegated notification outcomes are recorded in the bounded local notification
|
||||||
Never store recipient values, rendered body, credentials, or full provider
|
ring. `/notifications` renders selfcheck and delegated attempts without storing
|
||||||
responses in SQLite. Verify `/notifications` shows both selfcheck and delegated
|
recipient values, rendered body, credentials, provider responses, or errors.
|
||||||
attempts without leaking secrets.
|
|
||||||
|
|
||||||
### N3: Per-Account Webhook And Mattermost Credentials
|
### N3: Per-Account Webhook And Mattermost Credentials
|
||||||
|
|
||||||
|
|||||||
@@ -27,8 +27,9 @@ absolute lifetime. State-changing browser requests require CSRF validation.
|
|||||||
Authenticated responses use no-store and restrictive security headers.
|
Authenticated responses use no-store and restrictive security headers.
|
||||||
|
|
||||||
OAuth/device authorization is not implemented. Public exposure requires a TLS
|
OAuth/device authorization is not implemented. Public exposure requires a TLS
|
||||||
reverse proxy and network restrictions. Do not claim that `WORKER_URL` enables
|
reverse proxy and network restrictions. The accepted target name is
|
||||||
TLS; it only advertises the externally reachable URL.
|
`PUBLIC_URL`; `WORKER_URL` is current legacy configuration during migration.
|
||||||
|
Neither setting enables TLS; it only advertises the externally reachable URL.
|
||||||
|
|
||||||
## Known Runtime Gap
|
## Known Runtime Gap
|
||||||
|
|
||||||
|
|||||||
2
go.sum
2
go.sum
@@ -413,6 +413,8 @@ golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
|||||||
golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY=
|
golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY=
|
||||||
golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
||||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||||
|
golang.org/x/term v0.43.0 h1:S4RLU2sB31O/NCl+zFN9Aru9A/Cq2aqKpTZJ6B+DwT4=
|
||||||
|
golang.org/x/term v0.43.0/go.mod h1:lrhlHNdQJHO+1qVYiHfFKVuVioJIheAc3fBSMFYEIsk=
|
||||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||||
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
|
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
|
||||||
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||||
|
|||||||
@@ -27,6 +27,16 @@ type ExecutedCheck struct {
|
|||||||
Metrics []wire.MetricPoint
|
Metrics []wire.MetricPoint
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// SupportsKind reports whether Execute has a local executor for kind.
|
||||||
|
func SupportsKind(kind string) bool {
|
||||||
|
switch kind {
|
||||||
|
case "http", "ssl", "ssh", "ftp", "dns", "whois", "bssl", "llm", "llm-http", "ping", "tcp", "udp":
|
||||||
|
return true
|
||||||
|
default:
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Execute runs checks without saving to DB or InfluxDB.
|
// Execute runs checks without saving to DB or InfluxDB.
|
||||||
// Results are returned for reporting via API to the control plane.
|
// Results are returned for reporting via API to the control plane.
|
||||||
// This is designed for distributed workers that have no direct DB access.
|
// This is designed for distributed workers that have no direct DB access.
|
||||||
@@ -35,6 +45,9 @@ func Execute(m *models.Monitor, checks []models.Check) []ExecutedCheck {
|
|||||||
for i := range checks {
|
for i := range checks {
|
||||||
c := &checks[i]
|
c := &checks[i]
|
||||||
c.Monitor = m
|
c.Monitor = m
|
||||||
|
if !SupportsKind(c.Kind) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
switch c.Kind {
|
switch c.Kind {
|
||||||
case "http":
|
case "http":
|
||||||
r := chttp.Perform(c)
|
r := chttp.Perform(c)
|
||||||
|
|||||||
16
internal/checkexec/exec_test.go
Обычный файл
16
internal/checkexec/exec_test.go
Обычный файл
@@ -0,0 +1,16 @@
|
|||||||
|
package checkexec
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestSupportsKindMatchesExecuteDispatch(t *testing.T) {
|
||||||
|
for _, kind := range []string{"http", "ssl", "ssh", "ftp", "dns", "whois", "bssl", "llm", "llm-http", "ping", "tcp", "udp"} {
|
||||||
|
assert.Truef(t, SupportsKind(kind), "kind %q", kind)
|
||||||
|
}
|
||||||
|
for _, kind := range []string{"", "rkn", "smtp"} {
|
||||||
|
assert.Falsef(t, SupportsKind(kind), "kind %q", kind)
|
||||||
|
}
|
||||||
|
}
|
||||||
185
internal/compose/discover.go
Обычный файл
185
internal/compose/discover.go
Обычный файл
@@ -0,0 +1,185 @@
|
|||||||
|
package compose
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
// psItem is one NDJSON object from `docker ps --format json`. Only the
|
||||||
|
// fields discovery consumes are decoded; the rest are ignored. Field
|
||||||
|
// names match Docker's template JSON exactly (ID, Names, Image, State,
|
||||||
|
// Status, HealthStatus, Labels, Ports).
|
||||||
|
type psItem struct {
|
||||||
|
ID string `json:"ID"`
|
||||||
|
Names string `json:"Names"`
|
||||||
|
Image string `json:"Image"`
|
||||||
|
State string `json:"State"`
|
||||||
|
Status string `json:"Status"`
|
||||||
|
HealthStatus string `json:"HealthStatus"`
|
||||||
|
Labels string `json:"Labels"`
|
||||||
|
Ports string `json:"Ports"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Discover collects the current Compose project snapshot. It is the
|
||||||
|
// single entry point for the refresh loop and the operator console.
|
||||||
|
//
|
||||||
|
// The flow is:
|
||||||
|
// 1. `docker compose ls --all --format json` seeds the project map so
|
||||||
|
// exited/stopped projects (which have no running containers) still
|
||||||
|
// appear with their status and compose file.
|
||||||
|
// 2. `docker ps --format json` lists every running container; those
|
||||||
|
// carrying `com.docker.compose.project` are inspected and grouped.
|
||||||
|
// 3. Project-level metadata from container labels refines (and usually
|
||||||
|
// matches) the `compose ls` data.
|
||||||
|
//
|
||||||
|
// Docker being unavailable is reported as a non-empty Errors slice with
|
||||||
|
// an empty Projects map, so the caller renders a banner rather than a
|
||||||
|
// blank page. Per-container inspect failures are skipped silently (the
|
||||||
|
// container may have been removed mid-scan) and never abort the scan.
|
||||||
|
func Discover(ctx context.Context) (*DiscoveryResult, error) {
|
||||||
|
res := &DiscoveryResult{Projects: map[string]Project{}}
|
||||||
|
if !Available(ctx) {
|
||||||
|
res.Errors = append(res.Errors, "docker daemon is not available")
|
||||||
|
return res, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
ls := composeLS(ctx)
|
||||||
|
for name, item := range ls {
|
||||||
|
res.Projects[name] = Project{
|
||||||
|
Name: name,
|
||||||
|
Status: item.Status,
|
||||||
|
ConfigFiles: item.ConfigFiles,
|
||||||
|
Services: map[string]Service{},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
items, err := psItems(ctx)
|
||||||
|
if err != nil {
|
||||||
|
res.Errors = append(res.Errors, err.Error())
|
||||||
|
return res, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
containers, err := enrichContainers(ctx, items)
|
||||||
|
if err != nil {
|
||||||
|
// enrichContainers never returns a hard error today; keep the
|
||||||
|
// signature so future enrichment steps can report partial loss.
|
||||||
|
res.Errors = append(res.Errors, err.Error())
|
||||||
|
}
|
||||||
|
groupContainers(res, containers)
|
||||||
|
return res, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// psItems runs `docker ps --format json` and decodes the NDJSON stream.
|
||||||
|
// Each line is one container; malformed lines are skipped.
|
||||||
|
func psItems(ctx context.Context) ([]psItem, error) {
|
||||||
|
out, err := runDocker(ctx, "ps", "--format", "json")
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("docker ps: %w", err)
|
||||||
|
}
|
||||||
|
var items []psItem
|
||||||
|
for _, line := range strings.Split(strings.TrimSpace(out), "\n") {
|
||||||
|
line = strings.TrimSpace(line)
|
||||||
|
if line == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
var it psItem
|
||||||
|
if err := json.Unmarshal([]byte(line), &it); err != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
items = append(items, it)
|
||||||
|
}
|
||||||
|
return items, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// enrichContainers maps `docker ps` rows into enriched Container values.
|
||||||
|
// Containers without a Compose project label are skipped. Each retained
|
||||||
|
// container is `docker inspect`-ed for its host PID, mounts, published
|
||||||
|
// ports, and full label set; inspect failures downgrade to the `ps` data
|
||||||
|
// so a transient inspect error never drops a known container.
|
||||||
|
func enrichContainers(ctx context.Context, items []psItem) ([]Container, error) {
|
||||||
|
containers := make([]Container, 0, len(items))
|
||||||
|
for _, it := range items {
|
||||||
|
labels := parseLabels(it.Labels)
|
||||||
|
if labels[LabelComposeProject] == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
c := Container{
|
||||||
|
ID: it.ID,
|
||||||
|
Name: strings.TrimPrefix(it.Names, "/"),
|
||||||
|
Image: it.Image,
|
||||||
|
State: it.State,
|
||||||
|
Status: it.Status,
|
||||||
|
Health: it.HealthStatus,
|
||||||
|
Labels: labels,
|
||||||
|
}
|
||||||
|
if data, err := inspectContainer(ctx, it.ID); err == nil && data != nil {
|
||||||
|
c.PID = data.State.Pid
|
||||||
|
c.Ports = extractPorts(data)
|
||||||
|
c.Mounts = extractMounts(data)
|
||||||
|
for k, v := range data.Config.Labels {
|
||||||
|
c.Labels[k] = v
|
||||||
|
}
|
||||||
|
}
|
||||||
|
containers = append(containers, c)
|
||||||
|
}
|
||||||
|
return containers, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// groupContainers folds enriched containers into the project map. The
|
||||||
|
// project and service come from the Compose labels; the working dir and
|
||||||
|
// config file paths are lifted from any container in the project (they
|
||||||
|
// are identical across containers of one project) and override the
|
||||||
|
// `compose ls` values when present, since labels carry the exact paths
|
||||||
|
// the project was deployed with.
|
||||||
|
//
|
||||||
|
// Pure (no I/O) so it can be unit-tested with fixture containers.
|
||||||
|
func groupContainers(res *DiscoveryResult, containers []Container) {
|
||||||
|
for _, c := range containers {
|
||||||
|
projectName := c.Labels[LabelComposeProject]
|
||||||
|
if projectName == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
serviceName := c.Labels[LabelComposeService]
|
||||||
|
if serviceName == "" {
|
||||||
|
serviceName = c.Name
|
||||||
|
}
|
||||||
|
c.Service = serviceName
|
||||||
|
|
||||||
|
project, ok := res.Projects[projectName]
|
||||||
|
if !ok {
|
||||||
|
project = Project{Name: projectName, Services: map[string]Service{}}
|
||||||
|
}
|
||||||
|
// Container labels carry the authoritative working dir / config
|
||||||
|
// file; prefer them over the `compose ls` row when present.
|
||||||
|
if wd := c.Labels[LabelComposeWorkingDir]; wd != "" {
|
||||||
|
project.WorkingDir = wd
|
||||||
|
}
|
||||||
|
if cf := c.Labels[LabelComposeConfigFiles]; cf != "" {
|
||||||
|
project.ConfigFiles = cf
|
||||||
|
}
|
||||||
|
|
||||||
|
service := project.Services[serviceName]
|
||||||
|
service.Name = serviceName
|
||||||
|
service.Containers = append(service.Containers, c)
|
||||||
|
project.Services[serviceName] = service
|
||||||
|
|
||||||
|
res.Projects[projectName] = project
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Snapshot is a sorted, JSON-friendly view of a DiscoveryResult for the
|
||||||
|
// operator console: projects ordered by name. It is the shape rendered
|
||||||
|
// on /compose and returned by GET /web/api/compose.
|
||||||
|
type Snapshot struct {
|
||||||
|
Projects []ProjectSummary `json:"projects"`
|
||||||
|
Errors []string `json:"errors,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Summarize builds the sorted Snapshot from a raw DiscoveryResult.
|
||||||
|
func Summarize(res *DiscoveryResult) Snapshot {
|
||||||
|
snap := Snapshot{Errors: append([]string(nil), res.Errors...)}
|
||||||
|
snap.Projects = BuildAllProjectSummaries(res)
|
||||||
|
return snap
|
||||||
|
}
|
||||||
144
internal/compose/discover_test.go
Обычный файл
144
internal/compose/discover_test.go
Обычный файл
@@ -0,0 +1,144 @@
|
|||||||
|
package compose
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestParseLabels(t *testing.T) {
|
||||||
|
got := parseLabels("com.docker.compose.project=rsmon,com.docker.compose.service=web,foo=a=b")
|
||||||
|
if got[LabelComposeProject] != "rsmon" {
|
||||||
|
t.Fatalf("project = %q", got[LabelComposeProject])
|
||||||
|
}
|
||||||
|
if got[LabelComposeService] != "web" {
|
||||||
|
t.Fatalf("service = %q", got[LabelComposeService])
|
||||||
|
}
|
||||||
|
if got["foo"] != "a=b" { // value may contain '='
|
||||||
|
t.Fatalf("foo = %q", got["foo"])
|
||||||
|
}
|
||||||
|
if len(parseLabels("")) != 0 {
|
||||||
|
t.Fatal("empty labels should yield empty map")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestGroupContainers(t *testing.T) {
|
||||||
|
res := &DiscoveryResult{Projects: map[string]Project{
|
||||||
|
"rsmon": {Name: "rsmon", Status: "running(2)", ConfigFiles: "/compose/rsmon/docker-compose.yml", Services: map[string]Service{}},
|
||||||
|
}}
|
||||||
|
containers := []Container{
|
||||||
|
{
|
||||||
|
ID: "a",
|
||||||
|
Name: "rsmon-web",
|
||||||
|
Image: "nginx",
|
||||||
|
State: "running",
|
||||||
|
Labels: map[string]string{LabelComposeProject: "rsmon", LabelComposeService: "web", LabelComposeWorkingDir: "/compose/rsmon", LabelComposeConfigFiles: "/compose/rsmon/docker-compose.yml"},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
ID: "b",
|
||||||
|
Name: "rsmon-backend",
|
||||||
|
Image: "rsmon",
|
||||||
|
State: "running",
|
||||||
|
Labels: map[string]string{LabelComposeProject: "rsmon", LabelComposeService: "backend"},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
ID: "c",
|
||||||
|
Name: "loose",
|
||||||
|
State: "running",
|
||||||
|
Labels: map[string]string{}, // not a compose container
|
||||||
|
},
|
||||||
|
}
|
||||||
|
groupContainers(res, containers)
|
||||||
|
|
||||||
|
proj, ok := res.Projects["rsmon"]
|
||||||
|
if !ok {
|
||||||
|
t.Fatal("rsmon project missing")
|
||||||
|
}
|
||||||
|
if proj.WorkingDir != "/compose/rsmon" {
|
||||||
|
t.Fatalf("working dir = %q", proj.WorkingDir)
|
||||||
|
}
|
||||||
|
if len(proj.Services) != 2 {
|
||||||
|
t.Fatalf("service count = %d", len(proj.Services))
|
||||||
|
}
|
||||||
|
if got := proj.Services["web"].Containers[0].Service; got != "web" {
|
||||||
|
t.Fatalf("web service tag = %q", got)
|
||||||
|
}
|
||||||
|
if _, ok := res.Projects["loose"]; ok {
|
||||||
|
t.Fatal("non-compose container should not create a project")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseTraefikLabels(t *testing.T) {
|
||||||
|
routes := ParseTraefikLabels(map[string]string{
|
||||||
|
"traefik.http.routers.rsmon.rule": "Host(`rsmon.rscx.ru`)",
|
||||||
|
"traefik.http.routers.rsmon-api.rule": "PathPrefix(`/api`)",
|
||||||
|
"traefik.http.routers.rsmon.service": "rsmon", // non-rule label, ignored
|
||||||
|
"traefik.http.middlewares.redirect.rule": "not a router", // wrong prefix path
|
||||||
|
})
|
||||||
|
if len(routes) != 2 {
|
||||||
|
t.Fatalf("route count = %d", len(routes))
|
||||||
|
}
|
||||||
|
var hosts, prefixes []string
|
||||||
|
for _, r := range routes {
|
||||||
|
hosts = append(hosts, r.Hostnames...)
|
||||||
|
prefixes = append(prefixes, r.PathPrefixes...)
|
||||||
|
}
|
||||||
|
if !contains(hosts, "rsmon.rscx.ru") {
|
||||||
|
t.Fatalf("hosts = %v", hosts)
|
||||||
|
}
|
||||||
|
if !contains(prefixes, "/api") {
|
||||||
|
t.Fatalf("prefixes = %v", prefixes)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func contains(s []string, v string) bool {
|
||||||
|
for _, x := range s {
|
||||||
|
if x == v {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBuildProjectSummary(t *testing.T) {
|
||||||
|
res := &DiscoveryResult{Projects: map[string]Project{
|
||||||
|
"p": {
|
||||||
|
Name: "p",
|
||||||
|
Services: map[string]Service{
|
||||||
|
"web": {Name: "web", Containers: []Container{
|
||||||
|
{Name: "web-1", State: "running", Image: "nginx", Ports: []PortBinding{{HostPort: "80"}}},
|
||||||
|
}},
|
||||||
|
"db": {Name: "db", Containers: []Container{
|
||||||
|
{Name: "db-1", State: "exited", Image: "postgres"},
|
||||||
|
}},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}}
|
||||||
|
snap := Summarize(res)
|
||||||
|
if len(snap.Projects) != 1 {
|
||||||
|
t.Fatalf("project count = %d", len(snap.Projects))
|
||||||
|
}
|
||||||
|
p := snap.Projects[0]
|
||||||
|
if p.ContainerCount != 2 || p.RunningCount != 1 || p.ServiceCount != 2 {
|
||||||
|
t.Fatalf("counts = svc=%d c=%d run=%d", p.ServiceCount, p.ContainerCount, p.RunningCount)
|
||||||
|
}
|
||||||
|
if p.Services[0].Name != "db" { // sorted alphabetically
|
||||||
|
t.Fatalf("first service = %q", p.Services[0].Name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestValidProjectAction(t *testing.T) {
|
||||||
|
for _, a := range []string{"up", "down", "stop", "restart", "pull"} {
|
||||||
|
if _, ok := ValidProjectAction(a); !ok {
|
||||||
|
t.Errorf("%q should be valid", a)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if _, ok := ValidProjectAction("rm"); ok {
|
||||||
|
t.Error("rm should be rejected")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestManageProjectRejectsUnknownAction(t *testing.T) {
|
||||||
|
if _, err := ManageProject(context.Background(), ProjectRefs{Name: "x", WorkingDir: "/tmp"}, "bogus"); err == nil {
|
||||||
|
t.Fatal("expected error for unknown action")
|
||||||
|
}
|
||||||
|
}
|
||||||
130
internal/compose/docker.go
Обычный файл
130
internal/compose/docker.go
Обычный файл
@@ -0,0 +1,130 @@
|
|||||||
|
package compose
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"os/exec"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// dockerBin is the Docker CLI binary name. Overridable in tests via
|
||||||
|
// SetDockerBin so the management/discovery code never hard-codes a path.
|
||||||
|
var dockerBin = "docker"
|
||||||
|
|
||||||
|
// SetDockerBin overrides the Docker CLI binary used by discovery and
|
||||||
|
// management. Pass an empty value to restore the default ("docker").
|
||||||
|
// Tests use it to point at a stub binary; production leaves it alone.
|
||||||
|
func SetDockerBin(name string) {
|
||||||
|
if name == "" {
|
||||||
|
dockerBin = "docker"
|
||||||
|
return
|
||||||
|
}
|
||||||
|
dockerBin = name
|
||||||
|
}
|
||||||
|
|
||||||
|
// commandTimeout is the deadline applied to every Docker CLI call so a
|
||||||
|
// hung dockerd cannot wedge the inventory refresh loop or a management
|
||||||
|
// request. 30s is well above `docker ps` / `docker inspect` on a busy
|
||||||
|
// host but short enough that the operator notices.
|
||||||
|
const commandTimeout = 30 * time.Second
|
||||||
|
|
||||||
|
// Available reports whether the Docker CLI can reach the daemon. Used
|
||||||
|
// by the refresh loop to no-op (instead of erroring) on hosts without
|
||||||
|
// Docker, and by the UI to show a "Docker not available" banner.
|
||||||
|
func Available(ctx context.Context) bool {
|
||||||
|
c, cancel := context.WithTimeout(ctx, commandTimeout)
|
||||||
|
defer cancel()
|
||||||
|
if err := exec.CommandContext(c, dockerBin, "version").Run(); err != nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
// `docker version` succeeds against the client even when the daemon
|
||||||
|
// is down on some builds; `docker ps` requires the daemon, so it is
|
||||||
|
// the authoritative liveness probe.
|
||||||
|
pc, pcancel := context.WithTimeout(ctx, commandTimeout)
|
||||||
|
defer pcancel()
|
||||||
|
return exec.CommandContext(pc, dockerBin, "ps", "--format", "{{.ID}}").Run() == nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// composeLSItem is one row of `docker compose ls --all --format json`.
|
||||||
|
type composeLSItem struct {
|
||||||
|
Name string `json:"Name"`
|
||||||
|
Status string `json:"Status"`
|
||||||
|
ConfigFiles string `json:"ConfigFiles"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// composeLS runs `docker compose ls --all --format json` and returns
|
||||||
|
// the projects keyed by name. Unlike `docker ps --format json` (NDJSON),
|
||||||
|
// `docker compose ls` emits a single JSON array, so the whole stdout is
|
||||||
|
// unmarshalled at once. Non-fatal: returns an empty map on any error.
|
||||||
|
func composeLS(ctx context.Context) map[string]composeLSItem {
|
||||||
|
out, err := runDocker(ctx, "compose", "ls", "--all", "--format", "json")
|
||||||
|
if err != nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
trimmed := strings.TrimSpace(out)
|
||||||
|
if trimmed == "" {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
var items []composeLSItem
|
||||||
|
if err := json.Unmarshal([]byte(trimmed), &items); err != nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
m := make(map[string]composeLSItem, len(items))
|
||||||
|
for _, it := range items {
|
||||||
|
m[it.Name] = it
|
||||||
|
}
|
||||||
|
return m
|
||||||
|
}
|
||||||
|
|
||||||
|
// runDocker executes the Docker CLI with the given args and returns
|
||||||
|
// stdout. The context deadline is enforced by Go (CommandContext); a
|
||||||
|
// non-zero exit yields an error that carries the daemon's stderr so the
|
||||||
|
// caller can surface the daemon's diagnostic instead of a bare exit code.
|
||||||
|
func runDocker(ctx context.Context, args ...string) (string, error) {
|
||||||
|
c, cancel := context.WithTimeout(ctx, commandTimeout)
|
||||||
|
defer cancel()
|
||||||
|
cmd := exec.CommandContext(c, dockerBin, args...)
|
||||||
|
out, err := cmd.Output()
|
||||||
|
if err != nil {
|
||||||
|
if ee, ok := err.(*exec.ExitError); ok && len(ee.Stderr) > 0 {
|
||||||
|
return "", fmt.Errorf("%s: %s", strings.Join(args, " "), strings.TrimSpace(string(ee.Stderr)))
|
||||||
|
}
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
return string(out), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// dockerCombined runs the Docker CLI with a working directory and
|
||||||
|
// returns combined stdout+stderr. Used by management operations, which
|
||||||
|
// must run inside the project's working directory so Compose resolves
|
||||||
|
// the compose file and `.env` exactly as the operator would. Unlike
|
||||||
|
// runDocker, stderr is merged into the returned string because compose
|
||||||
|
// progress output goes to stderr and is useful to the operator.
|
||||||
|
func dockerCombined(ctx context.Context, dir string, args ...string) (string, error) {
|
||||||
|
c, cancel := context.WithTimeout(ctx, commandTimeout)
|
||||||
|
defer cancel()
|
||||||
|
cmd := exec.CommandContext(c, dockerBin, args...)
|
||||||
|
cmd.Dir = dir
|
||||||
|
out, err := cmd.CombinedOutput()
|
||||||
|
return string(out), err
|
||||||
|
}
|
||||||
|
|
||||||
|
// parseLabels parses the comma-separated `key=value` label string that
|
||||||
|
// `docker ps --format json` emits in the Labels field. Tolerates empty
|
||||||
|
// input and values containing '=' (only the first '=' splits).
|
||||||
|
func parseLabels(labels string) map[string]string {
|
||||||
|
out := make(map[string]string)
|
||||||
|
if labels == "" {
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
for _, pair := range strings.Split(labels, ",") {
|
||||||
|
k, v, ok := strings.Cut(pair, "=")
|
||||||
|
if !ok {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
out[strings.TrimSpace(k)] = strings.TrimSpace(v)
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
94
internal/compose/inspect.go
Обычный файл
94
internal/compose/inspect.go
Обычный файл
@@ -0,0 +1,94 @@
|
|||||||
|
package compose
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
// inspectData mirrors the subset of `docker inspect <id>` output that
|
||||||
|
// discovery consumes. Env is intentionally absent: the worker must not
|
||||||
|
// collect container environment (which carries secrets). Only Config
|
||||||
|
// labels, runtime State (for the host PID), Mounts, and published Ports
|
||||||
|
// are read.
|
||||||
|
type inspectData struct {
|
||||||
|
ID string `json:"Id"`
|
||||||
|
State inspectState `json:"State"`
|
||||||
|
Config inspectConfig `json:"Config"`
|
||||||
|
Mounts []inspectMount `json:"Mounts"`
|
||||||
|
NetworkSettings inspectNetworks `json:"NetworkSettings"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type inspectState struct {
|
||||||
|
Status string `json:"Status"`
|
||||||
|
Running bool `json:"Running"`
|
||||||
|
Pid int `json:"Pid"`
|
||||||
|
ExitCode int `json:"ExitCode"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type inspectConfig struct {
|
||||||
|
Labels map[string]string `json:"Labels"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type inspectMount struct {
|
||||||
|
Type string `json:"Type"`
|
||||||
|
Source string `json:"Source"`
|
||||||
|
Destination string `json:"Destination"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type inspectNetworks struct {
|
||||||
|
Ports portMap `json:"Ports"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// portMap mirrors the Docker inspect "Ports" object: the key is the
|
||||||
|
// container port ("80/tcp") and the value is the list of host bindings
|
||||||
|
// (nil when the port is exposed but not published).
|
||||||
|
type portMap map[string][]portBinding
|
||||||
|
|
||||||
|
type portBinding struct {
|
||||||
|
HostIP string `json:"HostIp"`
|
||||||
|
HostPort string `json:"HostPort"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// inspectContainer runs `docker inspect <id>` and decodes the first
|
||||||
|
// element. A missing container (race with `docker rm`) is reported as
|
||||||
|
// an error so the caller can skip it without aborting the whole scan.
|
||||||
|
func inspectContainer(ctx context.Context, id string) (*inspectData, error) {
|
||||||
|
out, err := runDocker(ctx, "inspect", id)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
var data []inspectData
|
||||||
|
if err := json.Unmarshal([]byte(strings.TrimSpace(out)), &data); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if len(data) == 0 {
|
||||||
|
return nil, fmt.Errorf("no inspect data for %s", id)
|
||||||
|
}
|
||||||
|
return &data[0], nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// extractPorts flattens the inspect Ports map into a slice of host
|
||||||
|
// bindings. Container ports that are exposed but not published (nil
|
||||||
|
// binding list) are skipped because they have no host-side footprint.
|
||||||
|
func extractPorts(data *inspectData) []PortBinding {
|
||||||
|
var ports []PortBinding
|
||||||
|
for _, bindings := range data.NetworkSettings.Ports {
|
||||||
|
for _, b := range bindings {
|
||||||
|
ports = append(ports, PortBinding{HostIP: b.HostIP, HostPort: b.HostPort})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return ports
|
||||||
|
}
|
||||||
|
|
||||||
|
// extractMounts copies the inspect Mounts into the wire Mount shape,
|
||||||
|
// dropping Docker-internal fields (mode, propagation, rw) the operator
|
||||||
|
// console does not render.
|
||||||
|
func extractMounts(data *inspectData) []Mount {
|
||||||
|
mounts := make([]Mount, 0, len(data.Mounts))
|
||||||
|
for _, m := range data.Mounts {
|
||||||
|
mounts = append(mounts, Mount{Source: m.Source, Destination: m.Destination, Type: m.Type})
|
||||||
|
}
|
||||||
|
return mounts
|
||||||
|
}
|
||||||
124
internal/compose/manage.go
Обычный файл
124
internal/compose/manage.go
Обычный файл
@@ -0,0 +1,124 @@
|
|||||||
|
package compose
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Action is a management operation the operator console can run against
|
||||||
|
// a project. The string values are the suffix used in the management
|
||||||
|
// API path (/compose/{project}/{action}) and the audit log target.
|
||||||
|
type Action string
|
||||||
|
|
||||||
|
const (
|
||||||
|
ActionUp Action = "up"
|
||||||
|
ActionDown Action = "down"
|
||||||
|
ActionStop Action = "stop"
|
||||||
|
ActionRestart Action = "restart"
|
||||||
|
ActionPull Action = "pull"
|
||||||
|
ActionLogs Action = "logs"
|
||||||
|
)
|
||||||
|
|
||||||
|
// projectActions is the allowlist of actions a project-level POST
|
||||||
|
// accepts. It maps the action to the compose subcommand(s). The map is
|
||||||
|
// the single source of truth so the route handler and the executor
|
||||||
|
// agree on what is permitted.
|
||||||
|
var projectActions = map[Action][]string{
|
||||||
|
ActionUp: {"up", "-d", "--remove-orphans"},
|
||||||
|
ActionDown: {"down", "--remove-orphans"},
|
||||||
|
ActionStop: {"stop"},
|
||||||
|
ActionRestart: {"restart"},
|
||||||
|
ActionPull: {"pull"},
|
||||||
|
}
|
||||||
|
|
||||||
|
// serviceActions is the allowlist for service-level operations
|
||||||
|
// (start/stop/restart a single service inside a project).
|
||||||
|
var serviceActions = map[Action][]string{
|
||||||
|
ActionUp: {"up", "-d", "--no-deps"},
|
||||||
|
ActionStop: {"stop"},
|
||||||
|
ActionRestart: {"restart"},
|
||||||
|
}
|
||||||
|
|
||||||
|
// ValidProjectAction reports whether name is an accepted project action.
|
||||||
|
func ValidProjectAction(name string) (Action, bool) {
|
||||||
|
a := Action(name)
|
||||||
|
_, ok := projectActions[a]
|
||||||
|
if !ok && a == ActionLogs {
|
||||||
|
return a, true
|
||||||
|
}
|
||||||
|
return a, ok
|
||||||
|
}
|
||||||
|
|
||||||
|
// ValidServiceAction reports whether name is an accepted service action.
|
||||||
|
func ValidServiceAction(name string) (Action, bool) {
|
||||||
|
a := Action(name)
|
||||||
|
_, ok := serviceActions[a]
|
||||||
|
return a, ok
|
||||||
|
}
|
||||||
|
|
||||||
|
// ManagementResult is returned by every management operation. Output is
|
||||||
|
// the combined compose stdout/stderr; OK is false when the command
|
||||||
|
// exited non-zero. The handler serializes it as JSON for the API path
|
||||||
|
// and renders Output in a <pre> block on the HTML path.
|
||||||
|
type ManagementResult struct {
|
||||||
|
OK bool `json:"ok"`
|
||||||
|
Action string `json:"action"`
|
||||||
|
Output string `json:"output"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// ManageProject runs an action against a whole project. refs supplies
|
||||||
|
// the working directory (so Compose auto-discovers the compose file and
|
||||||
|
// .env) and the project name (pinned with -p). A missing working dir is
|
||||||
|
// a hard error: without it Compose cannot locate the project files.
|
||||||
|
func ManageProject(ctx context.Context, refs ProjectRefs, action Action) (ManagementResult, error) {
|
||||||
|
args, ok := projectActions[action]
|
||||||
|
if !ok {
|
||||||
|
return ManagementResult{}, fmt.Errorf("compose: unknown project action %q", action)
|
||||||
|
}
|
||||||
|
return runCompose(ctx, refs, args...)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ManageService runs an action against a single service in a project.
|
||||||
|
// --no-deps on `up` ensures starting one service does not implicitly
|
||||||
|
// recreate its dependencies (matching dockge's service-start behavior).
|
||||||
|
func ManageService(ctx context.Context, refs ProjectRefs, service string, action Action) (ManagementResult, error) {
|
||||||
|
args, ok := serviceActions[action]
|
||||||
|
if !ok {
|
||||||
|
return ManagementResult{}, fmt.Errorf("compose: unknown service action %q", action)
|
||||||
|
}
|
||||||
|
if service == "" {
|
||||||
|
return ManagementResult{}, fmt.Errorf("compose: service is required")
|
||||||
|
}
|
||||||
|
full := append(append([]string{}, args...), service)
|
||||||
|
return runCompose(ctx, refs, full...)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Logs returns the recent log output for a project as a single string.
|
||||||
|
// It runs `docker compose logs --no-color --tail <n>` (no -f) so the
|
||||||
|
// operator console gets a finite snapshot; streaming tails are a later
|
||||||
|
// enhancement (dockge uses a PTY + websocket, out of scope here).
|
||||||
|
func Logs(ctx context.Context, refs ProjectRefs, tail int) (ManagementResult, error) {
|
||||||
|
if tail <= 0 {
|
||||||
|
tail = 200
|
||||||
|
}
|
||||||
|
return runCompose(ctx, refs, "logs", "--no-color", "--tail", fmt.Sprintf("%d", tail))
|
||||||
|
}
|
||||||
|
|
||||||
|
// runCompose executes `docker compose -p <name> <args...>` with the
|
||||||
|
// working directory set to refs.WorkingDir. The working dir makes
|
||||||
|
// Compose resolve the compose file and `.env` exactly as the operator
|
||||||
|
// would from the shell; -p pins the project name so the command cannot
|
||||||
|
// accidentally target a different project that shares the directory.
|
||||||
|
func runCompose(ctx context.Context, refs ProjectRefs, args ...string) (ManagementResult, error) {
|
||||||
|
if refs.WorkingDir == "" {
|
||||||
|
return ManagementResult{OK: false, Output: "compose project working directory is unknown; cannot manage"}, fmt.Errorf("compose: empty working dir")
|
||||||
|
}
|
||||||
|
full := append([]string{"compose", "-p", refs.Name}, args...)
|
||||||
|
out, err := dockerCombined(ctx, refs.WorkingDir, full...)
|
||||||
|
res := ManagementResult{OK: err == nil, Action: strings.Join(args, " "), Output: out}
|
||||||
|
if err != nil {
|
||||||
|
return res, err
|
||||||
|
}
|
||||||
|
return res, nil
|
||||||
|
}
|
||||||
199
internal/compose/summary.go
Обычный файл
199
internal/compose/summary.go
Обычный файл
@@ -0,0 +1,199 @@
|
|||||||
|
package compose
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
// ProjectSummary is the operator-facing view of one Compose project:
|
||||||
|
// counts, per-service containers, grouped mounts, and Traefik routes.
|
||||||
|
// It is the shape rendered on /compose and returned by the JSON API.
|
||||||
|
type ProjectSummary struct {
|
||||||
|
Name string `json:"name"`
|
||||||
|
Status string `json:"status"`
|
||||||
|
ConfigFiles string `json:"config_files"`
|
||||||
|
WorkingDir string `json:"working_dir"`
|
||||||
|
ServiceCount int `json:"service_count"`
|
||||||
|
ContainerCount int `json:"container_count"`
|
||||||
|
RunningCount int `json:"running_count"`
|
||||||
|
Services []ServiceSummary `json:"services"`
|
||||||
|
GroupedMounts []ProjectMountInfo `json:"grouped_mounts"`
|
||||||
|
AllTraefikRoutes []ProjectTraefikRoute `json:"traefik_routes"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// ServiceSummary is a condensed view of a Compose service.
|
||||||
|
type ServiceSummary struct {
|
||||||
|
Name string `json:"name"`
|
||||||
|
Containers []ContainerSummary `json:"containers"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// ContainerSummary holds the key container fields the detail page shows.
|
||||||
|
type ContainerSummary struct {
|
||||||
|
Name string `json:"name"`
|
||||||
|
ID string `json:"id"`
|
||||||
|
Image string `json:"image"`
|
||||||
|
State string `json:"state"`
|
||||||
|
Status string `json:"status"`
|
||||||
|
Health string `json:"health"`
|
||||||
|
PID int `json:"pid"`
|
||||||
|
Ports []PortBinding `json:"ports"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// ProjectMountInfo represents a mount that may be shared across the
|
||||||
|
// services/containers of a project.
|
||||||
|
type ProjectMountInfo struct {
|
||||||
|
Source string `json:"source"`
|
||||||
|
Destination string `json:"destination"`
|
||||||
|
Type string `json:"type"`
|
||||||
|
UsedBy []MountUsage `json:"used_by"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// MountUsage identifies which service/container uses a mount.
|
||||||
|
type MountUsage struct {
|
||||||
|
Service string `json:"service"`
|
||||||
|
Container string `json:"container"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// ProjectTraefikRoute is a Traefik route attributed to the project.
|
||||||
|
type ProjectTraefikRoute struct {
|
||||||
|
RouterName string `json:"router_name"`
|
||||||
|
Service string `json:"service"`
|
||||||
|
Container string `json:"container"`
|
||||||
|
Hostnames []string `json:"hostnames"`
|
||||||
|
PathPrefixes []string `json:"path_prefixes"`
|
||||||
|
Paths []string `json:"paths"`
|
||||||
|
Rule string `json:"rule"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// BuildProjectSummary creates the operator-facing summary from a raw
|
||||||
|
// Project. Services, mounts, and routes are sorted for stable display.
|
||||||
|
func BuildProjectSummary(p Project) ProjectSummary {
|
||||||
|
s := ProjectSummary{
|
||||||
|
Name: p.Name,
|
||||||
|
Status: p.Status,
|
||||||
|
ConfigFiles: p.ConfigFiles,
|
||||||
|
WorkingDir: p.WorkingDir,
|
||||||
|
Services: make([]ServiceSummary, 0, len(p.Services)),
|
||||||
|
GroupedMounts: make([]ProjectMountInfo, 0),
|
||||||
|
AllTraefikRoutes: make([]ProjectTraefikRoute, 0),
|
||||||
|
}
|
||||||
|
mountGroups := map[string]*ProjectMountInfo{}
|
||||||
|
|
||||||
|
for svcName, svc := range p.Services {
|
||||||
|
s.ServiceCount++
|
||||||
|
svcSummary := ServiceSummary{Name: svcName, Containers: make([]ContainerSummary, 0, len(svc.Containers))}
|
||||||
|
for _, c := range svc.Containers {
|
||||||
|
s.ContainerCount++
|
||||||
|
if c.State == "running" {
|
||||||
|
s.RunningCount++
|
||||||
|
}
|
||||||
|
svcSummary.Containers = append(svcSummary.Containers, ContainerSummary{
|
||||||
|
Name: c.Name,
|
||||||
|
ID: c.ID,
|
||||||
|
Image: c.Image,
|
||||||
|
State: c.State,
|
||||||
|
Status: c.Status,
|
||||||
|
Health: c.Health,
|
||||||
|
PID: c.PID,
|
||||||
|
Ports: c.Ports,
|
||||||
|
})
|
||||||
|
groupMount(mountGroups, svcName, c)
|
||||||
|
for _, r := range ExtractTraefikRoutes(c) {
|
||||||
|
s.AllTraefikRoutes = append(s.AllTraefikRoutes, ProjectTraefikRoute{
|
||||||
|
RouterName: r.RouterName,
|
||||||
|
Service: svcName,
|
||||||
|
Container: c.Name,
|
||||||
|
Hostnames: r.Hostnames,
|
||||||
|
PathPrefixes: r.PathPrefixes,
|
||||||
|
Paths: r.Paths,
|
||||||
|
Rule: r.Rule,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
s.Services = append(s.Services, svcSummary)
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, mi := range mountGroups {
|
||||||
|
s.GroupedMounts = append(s.GroupedMounts, *mi)
|
||||||
|
}
|
||||||
|
sort.Slice(s.Services, func(i, j int) bool { return s.Services[i].Name < s.Services[j].Name })
|
||||||
|
sort.Slice(s.GroupedMounts, func(i, j int) bool { return s.GroupedMounts[i].Source < s.GroupedMounts[j].Source })
|
||||||
|
sort.Slice(s.AllTraefikRoutes, func(i, j int) bool {
|
||||||
|
return s.AllTraefikRoutes[i].RouterName < s.AllTraefikRoutes[j].RouterName
|
||||||
|
})
|
||||||
|
return s
|
||||||
|
}
|
||||||
|
|
||||||
|
func groupMount(groups map[string]*ProjectMountInfo, svc string, c Container) {
|
||||||
|
for _, m := range c.Mounts {
|
||||||
|
key := fmt.Sprintf("%s|%s|%s", m.Source, m.Destination, m.Type)
|
||||||
|
usage := MountUsage{Service: svc, Container: c.Name}
|
||||||
|
if existing := groups[key]; existing != nil {
|
||||||
|
for _, u := range existing.UsedBy {
|
||||||
|
if u == usage {
|
||||||
|
goto next
|
||||||
|
}
|
||||||
|
}
|
||||||
|
existing.UsedBy = append(existing.UsedBy, usage)
|
||||||
|
next:
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
groups[key] = &ProjectMountInfo{
|
||||||
|
Source: m.Source, Destination: m.Destination, Type: m.Type, UsedBy: []MountUsage{usage},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// BuildAllProjectSummaries returns every project summary sorted by name.
|
||||||
|
func BuildAllProjectSummaries(res *DiscoveryResult) []ProjectSummary {
|
||||||
|
out := make([]ProjectSummary, 0, len(res.Projects))
|
||||||
|
for _, p := range res.Projects {
|
||||||
|
out = append(out, BuildProjectSummary(p))
|
||||||
|
}
|
||||||
|
sort.Slice(out, func(i, j int) bool { return out[i].Name < out[j].Name })
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// FormatMountSourceShort shortens a long mount source for table display.
|
||||||
|
func FormatMountSourceShort(source string, maxLen int) string {
|
||||||
|
if len(source) <= maxLen || maxLen < 10 {
|
||||||
|
return source
|
||||||
|
}
|
||||||
|
keep := (maxLen - 3) / 2
|
||||||
|
return source[:keep] + "..." + source[len(source)-keep:]
|
||||||
|
}
|
||||||
|
|
||||||
|
// SharedMountSummary returns a short label describing who uses a mount.
|
||||||
|
func (m *ProjectMountInfo) SharedMountSummary() string {
|
||||||
|
if len(m.UsedBy) == 0 {
|
||||||
|
return "unused"
|
||||||
|
}
|
||||||
|
if len(m.UsedBy) == 1 {
|
||||||
|
return fmt.Sprintf("%s/%s", m.UsedBy[0].Service, m.UsedBy[0].Container)
|
||||||
|
}
|
||||||
|
counts := map[string]int{}
|
||||||
|
for _, u := range m.UsedBy {
|
||||||
|
counts[u.Service]++
|
||||||
|
}
|
||||||
|
var parts []string
|
||||||
|
for svc, n := range counts {
|
||||||
|
if n == 1 {
|
||||||
|
parts = append(parts, svc)
|
||||||
|
} else {
|
||||||
|
parts = append(parts, fmt.Sprintf("%s(%d)", svc, n))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
sort.Strings(parts)
|
||||||
|
return "shared: " + strings.Join(parts, ", ")
|
||||||
|
}
|
||||||
|
|
||||||
|
// FindProject returns the summary for the named project, or nil.
|
||||||
|
func (s Snapshot) FindProject(name string) *ProjectSummary {
|
||||||
|
for i := range s.Projects {
|
||||||
|
if s.Projects[i].Name == name {
|
||||||
|
return &s.Projects[i]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
87
internal/compose/traefik.go
Обычный файл
87
internal/compose/traefik.go
Обычный файл
@@ -0,0 +1,87 @@
|
|||||||
|
package compose
|
||||||
|
|
||||||
|
import (
|
||||||
|
"regexp"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Traefik label parsing. Compose projects fronted by Traefik encode
|
||||||
|
// their router rules as `traefik.http.routers.<name>.rule=Host(...)`.
|
||||||
|
// We surface hostnames/paths so the operator console can show which
|
||||||
|
// domains a project serves without reading raw labels.
|
||||||
|
|
||||||
|
const (
|
||||||
|
traefikLabelPrefix = "traefik.http.routers."
|
||||||
|
traefikRuleSuffix = ".rule"
|
||||||
|
)
|
||||||
|
|
||||||
|
var (
|
||||||
|
// Host(`example.com`) or Host("example.com")
|
||||||
|
hostRegex = regexp.MustCompile(`Host\s*\(\s*[` + "`" + `"]([^` + "`" + `"]+)[` + "`" + `"]\s*\)`)
|
||||||
|
// Path(`/api`) or Path("/api")
|
||||||
|
pathRegex = regexp.MustCompile(`Path\s*\(\s*[` + "`" + `"]([^` + "`" + `"]+)[` + "`" + `"]\s*\)`)
|
||||||
|
// PathPrefix(`/api`) or PathPrefix("/api")
|
||||||
|
pathPrefixRegex = regexp.MustCompile(`PathPrefix\s*\(\s*[` + "`" + `"]([^` + "`" + `"]+)[` + "`" + `"]\s*\)`)
|
||||||
|
)
|
||||||
|
|
||||||
|
// TraefikRoute is one decoded Traefik router derived from labels.
|
||||||
|
type TraefikRoute struct {
|
||||||
|
RouterName string `json:"router_name"`
|
||||||
|
Hostnames []string `json:"hostnames"`
|
||||||
|
Paths []string `json:"paths"`
|
||||||
|
PathPrefixes []string `json:"path_prefixes"`
|
||||||
|
Rule string `json:"rule"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// ParseTraefikLabels scans a label map for `traefik.http.routers.<n>.rule`
|
||||||
|
// entries and returns one TraefikRoute per router, with the rule's
|
||||||
|
// Host/Path/PathPrefix operands extracted. Pure function: no I/O.
|
||||||
|
func ParseTraefikLabels(labels map[string]string) []TraefikRoute {
|
||||||
|
routes := map[string]*TraefikRoute{}
|
||||||
|
for key, value := range labels {
|
||||||
|
if !strings.HasPrefix(key, traefikLabelPrefix) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
suffix := strings.TrimPrefix(key, traefikLabelPrefix)
|
||||||
|
idx := strings.Index(suffix, ".")
|
||||||
|
if idx == -1 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
routerName := suffix[:idx]
|
||||||
|
property := suffix[idx:]
|
||||||
|
if property != traefikRuleSuffix {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
route := routes[routerName]
|
||||||
|
if route == nil {
|
||||||
|
route = &TraefikRoute{RouterName: routerName}
|
||||||
|
}
|
||||||
|
route.Rule = value
|
||||||
|
route.Hostnames = extractAll(hostRegex, value)
|
||||||
|
route.Paths = extractAll(pathRegex, value)
|
||||||
|
route.PathPrefixes = extractAll(pathPrefixRegex, value)
|
||||||
|
routes[routerName] = route
|
||||||
|
}
|
||||||
|
out := make([]TraefikRoute, 0, len(routes))
|
||||||
|
for _, r := range routes {
|
||||||
|
out = append(out, *r)
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func extractAll(re *regexp.Regexp, rule string) []string {
|
||||||
|
matches := re.FindAllStringSubmatch(rule, -1)
|
||||||
|
out := make([]string, 0, len(matches))
|
||||||
|
for _, m := range matches {
|
||||||
|
if len(m) > 1 && m[1] != "" {
|
||||||
|
out = append(out, m[1])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// ExtractTraefikRoutes returns the Traefik routes declared by a
|
||||||
|
// container's labels.
|
||||||
|
func ExtractTraefikRoutes(c Container) []TraefikRoute {
|
||||||
|
return ParseTraefikLabels(c.Labels)
|
||||||
|
}
|
||||||
90
internal/compose/types.go
Обычный файл
90
internal/compose/types.go
Обычный файл
@@ -0,0 +1,90 @@
|
|||||||
|
// Package compose implements Docker Compose project discovery and
|
||||||
|
// management for the worker operator console.
|
||||||
|
//
|
||||||
|
// It talks to Docker exclusively through the `docker` / `docker compose`
|
||||||
|
// CLI over os/exec. There is no dependency on the Docker Engine SDK or
|
||||||
|
// compose-go: the CLI is the same surface deploymentd and dockge use, it
|
||||||
|
// needs no extra libraries, and it is already present on every host the
|
||||||
|
// worker manages. Discovery groups running containers by the
|
||||||
|
// `com.docker.compose.project` / `com.docker.compose.service` labels and
|
||||||
|
// merges the project-level metadata (status, compose file, working dir)
|
||||||
|
// reported by `docker compose ls`. Management runs `docker compose`
|
||||||
|
// against the project's working directory.
|
||||||
|
package compose
|
||||||
|
|
||||||
|
// Label constants written by Docker Compose onto every container it
|
||||||
|
// creates. They are the stable identity keys that map a raw container
|
||||||
|
// back to its Compose project and service.
|
||||||
|
const (
|
||||||
|
LabelComposeProject = "com.docker.compose.project"
|
||||||
|
LabelComposeService = "com.docker.compose.service"
|
||||||
|
LabelComposeWorkingDir = "com.docker.compose.project.working_dir"
|
||||||
|
LabelComposeConfigFiles = "com.docker.compose.project.config_files"
|
||||||
|
)
|
||||||
|
|
||||||
|
// PortBinding is a single host-side port mapping for a container.
|
||||||
|
type PortBinding struct {
|
||||||
|
HostIP string `json:"host_ip"`
|
||||||
|
HostPort string `json:"host_port"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Mount is a single bind/volume/tmpfs mount attached to a container.
|
||||||
|
type Mount struct {
|
||||||
|
Source string `json:"source"`
|
||||||
|
Destination string `json:"destination"`
|
||||||
|
Type string `json:"type"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Container is the per-container view discovery collects. It is the
|
||||||
|
// union of `docker ps` (name/image/state/status/health) and
|
||||||
|
// `docker inspect` (pid/ports/mounts/full labels).
|
||||||
|
type Container struct {
|
||||||
|
ID string `json:"id"`
|
||||||
|
Name string `json:"name"`
|
||||||
|
Image string `json:"image"`
|
||||||
|
State string `json:"state"`
|
||||||
|
Status string `json:"status"`
|
||||||
|
Health string `json:"health"`
|
||||||
|
PID int `json:"pid"`
|
||||||
|
Labels map[string]string `json:"labels"`
|
||||||
|
Ports []PortBinding `json:"ports"`
|
||||||
|
Mounts []Mount `json:"mounts"`
|
||||||
|
Service string `json:"service"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Service groups the containers that belong to one Compose service.
|
||||||
|
type Service struct {
|
||||||
|
Name string `json:"name"`
|
||||||
|
Containers []Container `json:"containers"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Project is a single Compose project. Status / ConfigFiles / WorkingDir
|
||||||
|
// come from `docker compose ls` and are refined from container labels
|
||||||
|
// (which carry the exact absolute paths the project was deployed with).
|
||||||
|
type Project struct {
|
||||||
|
Name string `json:"name"`
|
||||||
|
Status string `json:"status"`
|
||||||
|
ConfigFiles string `json:"config_files"`
|
||||||
|
WorkingDir string `json:"working_dir"`
|
||||||
|
Services map[string]Service `json:"services"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// DiscoveryResult is the full snapshot returned by Discover. Projects
|
||||||
|
// is keyed by project name. Errors carries non-fatal section failures
|
||||||
|
// (e.g. docker unavailable) so the UI can render a banner instead of a
|
||||||
|
// blank page.
|
||||||
|
type DiscoveryResult struct {
|
||||||
|
Projects map[string]Project `json:"projects"`
|
||||||
|
Errors []string `json:"errors,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// ProjectRefs resolves the Compose CLI arguments a management operation
|
||||||
|
// needs for a project: the working directory (so Compose auto-discovers
|
||||||
|
// the compose file and `.env`) and the project name (pinned with -p so
|
||||||
|
// the operation targets exactly this project even when the directory
|
||||||
|
// basename differs).
|
||||||
|
type ProjectRefs struct {
|
||||||
|
Name string
|
||||||
|
WorkingDir string
|
||||||
|
ConfigFiles string
|
||||||
|
}
|
||||||
@@ -2,12 +2,15 @@ package distworker
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
|
"context"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
|
"net"
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/url"
|
"net/url"
|
||||||
"strings"
|
"strings"
|
||||||
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/gorilla/websocket"
|
"github.com/gorilla/websocket"
|
||||||
@@ -49,12 +52,16 @@ func NewClient(endpoint, authToken string) *Client {
|
|||||||
|
|
||||||
// postJSON is a helper for sending JSON POST requests
|
// postJSON is a helper for sending JSON POST requests
|
||||||
func (c *Client) postJSON(path string, payload interface{}) (*http.Response, error) {
|
func (c *Client) postJSON(path string, payload interface{}) (*http.Response, error) {
|
||||||
|
return c.postJSONContext(context.Background(), path, payload)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Client) postJSONContext(ctx context.Context, path string, payload interface{}) (*http.Response, error) {
|
||||||
body, err := json.Marshal(payload)
|
body, err := json.Marshal(payload)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
httpReq, err := http.NewRequest("POST", c.endpoint+path, bytes.NewReader(body))
|
httpReq, err := http.NewRequestWithContext(ctx, "POST", c.endpoint+path, bytes.NewReader(body))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@@ -91,10 +98,10 @@ func (c *Client) Heartbeat(req wire.HeartbeatRequest) error {
|
|||||||
// bearer token for this worker. The current bearer is used for
|
// bearer token for this worker. The current bearer is used for
|
||||||
// authentication; the response carries the freshly issued token.
|
// authentication; the response carries the freshly issued token.
|
||||||
//
|
//
|
||||||
// Returns the new token string. The main app invalidates the old
|
// Returns the new token string. The main app invalidates the old token
|
||||||
// token immediately.
|
// immediately.
|
||||||
func (c *Client) RotateToken() (string, error) {
|
func (c *Client) RotateToken(ctx context.Context) (string, error) {
|
||||||
resp, err := c.postJSON("/api/internal/workers/rotate-token", struct{}{})
|
resp, err := c.postJSONContext(ctx, "/api/internal/workers/rotate-token", struct{}{})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", err
|
return "", err
|
||||||
}
|
}
|
||||||
@@ -115,6 +122,25 @@ func (c *Client) RotateToken() (string, error) {
|
|||||||
return out.AuthToken, nil
|
return out.AuthToken, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (c *Client) Bootstrap(ctx context.Context, workerID, token string) (wire.BootstrapResponse, error) {
|
||||||
|
var out wire.BootstrapResponse
|
||||||
|
resp, err := c.postJSONContext(ctx, "/api/internal/workers/bootstrap", wire.BootstrapRequest{WorkerID: workerID, BootstrapToken: token})
|
||||||
|
if err != nil {
|
||||||
|
return out, err
|
||||||
|
}
|
||||||
|
defer resp.Body.Close() //nolint:errcheck
|
||||||
|
if resp.StatusCode != http.StatusOK {
|
||||||
|
return out, checkStatusCode(resp, "bootstrap")
|
||||||
|
}
|
||||||
|
if err := json.NewDecoder(resp.Body).Decode(&out); err != nil {
|
||||||
|
return out, err
|
||||||
|
}
|
||||||
|
if out.AuthToken == "" || out.WorkerID != workerID || out.ConfigVerificationKey == "" {
|
||||||
|
return out, fmt.Errorf("invalid bootstrap response")
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
// GetJobs fetches available check jobs from the control plane
|
// GetJobs fetches available check jobs from the control plane
|
||||||
func (c *Client) GetJobs() (*wire.JobsResponse, error) {
|
func (c *Client) GetJobs() (*wire.JobsResponse, error) {
|
||||||
httpReq, err := http.NewRequest("GET", c.endpoint+"/api/internal/workers/jobs", http.NoBody)
|
httpReq, err := http.NewRequest("GET", c.endpoint+"/api/internal/workers/jobs", http.NoBody)
|
||||||
@@ -156,6 +182,12 @@ func (c *Client) ReportResults(req wire.ResultsRequest) error {
|
|||||||
|
|
||||||
// WorkerSocket connects to the websocket task channel.
|
// WorkerSocket connects to the websocket task channel.
|
||||||
func (c *Client) WorkerSocket() (*websocket.Conn, error) {
|
func (c *Client) WorkerSocket() (*websocket.Conn, error) {
|
||||||
|
return c.WorkerSocketContext(context.Background())
|
||||||
|
}
|
||||||
|
|
||||||
|
// WorkerSocketContext connects to the websocket task channel, cancelling the
|
||||||
|
// dial when the caller's context ends.
|
||||||
|
func (c *Client) WorkerSocketContext(ctx context.Context) (*websocket.Conn, error) {
|
||||||
endpoint := strings.TrimRight(c.endpoint, "/")
|
endpoint := strings.TrimRight(c.endpoint, "/")
|
||||||
wsURL := endpoint
|
wsURL := endpoint
|
||||||
if !strings.HasSuffix(wsURL, "/worker") && !strings.HasSuffix(wsURL, "/api/worker") {
|
if !strings.HasSuffix(wsURL, "/worker") && !strings.HasSuffix(wsURL, "/api/worker") {
|
||||||
@@ -175,7 +207,33 @@ func (c *Client) WorkerSocket() (*websocket.Conn, error) {
|
|||||||
q.Set("token", c.authToken)
|
q.Set("token", c.authToken)
|
||||||
u.RawQuery = q.Encode()
|
u.RawQuery = q.Encode()
|
||||||
|
|
||||||
conn, resp, err := websocket.DefaultDialer.Dial(u.String(), nil)
|
dialer := *websocket.DefaultDialer
|
||||||
|
var (
|
||||||
|
connMu sync.Mutex
|
||||||
|
dialConn net.Conn
|
||||||
|
)
|
||||||
|
stopClose := context.AfterFunc(ctx, func() {
|
||||||
|
connMu.Lock()
|
||||||
|
if dialConn != nil {
|
||||||
|
_ = dialConn.Close()
|
||||||
|
}
|
||||||
|
connMu.Unlock()
|
||||||
|
})
|
||||||
|
defer stopClose()
|
||||||
|
dialer.NetDialContext = func(dialCtx context.Context, network, address string) (net.Conn, error) {
|
||||||
|
conn, err := (&net.Dialer{}).DialContext(dialCtx, network, address)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
connMu.Lock()
|
||||||
|
dialConn = conn
|
||||||
|
if ctx.Err() != nil {
|
||||||
|
_ = conn.Close()
|
||||||
|
}
|
||||||
|
connMu.Unlock()
|
||||||
|
return conn, nil
|
||||||
|
}
|
||||||
|
conn, resp, err := dialer.DialContext(ctx, u.String(), nil)
|
||||||
if err != nil && resp != nil {
|
if err != nil && resp != nil {
|
||||||
defer resp.Body.Close() //nolint:errcheck
|
defer resp.Body.Close() //nolint:errcheck
|
||||||
body, _ := io.ReadAll(resp.Body)
|
body, _ := io.ReadAll(resp.Body)
|
||||||
|
|||||||
@@ -21,8 +21,18 @@ const (
|
|||||||
// same host. Operators are still free to override via WORKER_PORT.
|
// same host. Operators are still free to override via WORKER_PORT.
|
||||||
DefaultHTTPPort = 27401
|
DefaultHTTPPort = 27401
|
||||||
|
|
||||||
|
// EnvPublicURL is the canonical environment variable for the
|
||||||
|
// publicly advertised origin. It wins over the legacy
|
||||||
|
// EnvWorkerURLLegacy when both are set.
|
||||||
|
EnvPublicURL = "PUBLIC_URL"
|
||||||
|
// EnvWorkerURLLegacy is the legacy name for the advertised public
|
||||||
|
// origin. It is accepted only for the bounded migration defined in
|
||||||
|
// docs/public-endpoint-and-identity.md milestone 1 and is
|
||||||
|
// deprecated: new configuration must set EnvPublicURL.
|
||||||
|
EnvWorkerURLLegacy = "WORKER_URL"
|
||||||
|
|
||||||
// schemeHTTP / schemeHTTPS are the only schemes accepted on
|
// schemeHTTP / schemeHTTPS are the only schemes accepted on
|
||||||
// WORKER_URL. Peer workers and the main app need an http(s) origin
|
// PUBLIC_URL. Peer workers and the main app need an http(s) origin
|
||||||
// they can dial with Go's net/http stack.
|
// they can dial with Go's net/http stack.
|
||||||
schemeHTTP = "http"
|
schemeHTTP = "http"
|
||||||
schemeHTTPS = "https"
|
schemeHTTPS = "https"
|
||||||
@@ -34,22 +44,57 @@ const (
|
|||||||
|
|
||||||
// HTTPConfig holds the settings that govern the worker's local HTTP
|
// HTTPConfig holds the settings that govern the worker's local HTTP
|
||||||
// listener (web app MVP in Task 3 and Raft peer connections in Task 4).
|
// listener (web app MVP in Task 3 and Raft peer connections in Task 4).
|
||||||
// Host/Port are the bind interface. URL is the publicly-advertised
|
// Host/Port are the bind interface. PublicURL is the publicly-advertised
|
||||||
// location peers and the main app use to reach the worker; it is NOT
|
// origin peers and the main app use to reach the worker; it is NOT
|
||||||
// derived from Host:Port because workers commonly sit behind a reverse
|
// derived from Host:Port because workers commonly sit behind a reverse
|
||||||
// proxy / Traefik with HTTPS while listening on plain HTTP internally.
|
// proxy / Traefik with HTTPS while listening on plain HTTP internally.
|
||||||
//
|
//
|
||||||
|
// PublicURL is sourced from PUBLIC_URL (strict origin) or, for the
|
||||||
|
// bounded migration, from the deprecated WORKER_URL (legacy-tolerant
|
||||||
|
// absolute URL). URLSource records which variable supplied the value so
|
||||||
|
// validation and diagnostics name the correct source.
|
||||||
|
//
|
||||||
// Login/Password are basic auth credentials for the worker web app API
|
// Login/Password are basic auth credentials for the worker web app API
|
||||||
// (Task 3). They are kept in memory only; Task 3 may hash them before
|
// (Task 3). They are kept in memory only; Task 3 may hash them before
|
||||||
// any persistent store.
|
// any persistent store.
|
||||||
type HTTPConfig struct {
|
type HTTPConfig struct {
|
||||||
Host string
|
Host string
|
||||||
Port int
|
Port int
|
||||||
URL string
|
PublicURL string
|
||||||
|
URLSource PublicURLSource
|
||||||
Login string
|
Login string
|
||||||
Password string
|
Password string
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// PublicURLSource identifies which environment variable supplied the
|
||||||
|
// advertised public origin, so callers can log the legacy deprecation
|
||||||
|
// exactly once.
|
||||||
|
type PublicURLSource int
|
||||||
|
|
||||||
|
const (
|
||||||
|
// PublicURLSourceNone means no advertised origin is configured.
|
||||||
|
PublicURLSourceNone PublicURLSource = iota
|
||||||
|
// PublicURLSourceCanonical means PUBLIC_URL was configured.
|
||||||
|
PublicURLSourceCanonical
|
||||||
|
// PublicURLSourceLegacy means only WORKER_URL was configured; the
|
||||||
|
// value is accepted for the bounded migration.
|
||||||
|
PublicURLSourceLegacy
|
||||||
|
)
|
||||||
|
|
||||||
|
// PublicURLFromEnv reads the advertised public origin. PUBLIC_URL is
|
||||||
|
// the canonical source; the deprecated WORKER_URL is used only when
|
||||||
|
// PUBLIC_URL is unset. The returned source lets the caller warn about
|
||||||
|
// the legacy fallback without re-reading the environment.
|
||||||
|
func PublicURLFromEnv() (string, PublicURLSource) {
|
||||||
|
if v := strings.TrimSpace(os.Getenv(EnvPublicURL)); v != "" {
|
||||||
|
return v, PublicURLSourceCanonical
|
||||||
|
}
|
||||||
|
if v := strings.TrimSpace(os.Getenv(EnvWorkerURLLegacy)); v != "" {
|
||||||
|
return v, PublicURLSourceLegacy
|
||||||
|
}
|
||||||
|
return "", PublicURLSourceNone
|
||||||
|
}
|
||||||
|
|
||||||
// IsAuthConfigured reports whether both WORKER_LOGIN and WORKER_PASSWORD
|
// IsAuthConfigured reports whether both WORKER_LOGIN and WORKER_PASSWORD
|
||||||
// are set. The HTTP listener refuses to start otherwise.
|
// are set. The HTTP listener refuses to start otherwise.
|
||||||
func (c HTTPConfig) IsAuthConfigured() bool {
|
func (c HTTPConfig) IsAuthConfigured() bool {
|
||||||
@@ -68,6 +113,9 @@ func (c HTTPConfig) IsListenConfigured() bool {
|
|||||||
type Config struct {
|
type Config struct {
|
||||||
URL string
|
URL string
|
||||||
Token string
|
Token string
|
||||||
|
BootstrapToken string
|
||||||
|
StateFile string
|
||||||
|
WorkerID string
|
||||||
|
|
||||||
// MaxConcurrency caps the worker pool size and the number of dispatcher
|
// MaxConcurrency caps the worker pool size and the number of dispatcher
|
||||||
// goroutines. A value <= 0 falls back to DefaultMaxConcurrency.
|
// goroutines. A value <= 0 falls back to DefaultMaxConcurrency.
|
||||||
@@ -84,6 +132,9 @@ func ConfigFromEnv() Config {
|
|||||||
return Config{
|
return Config{
|
||||||
URL: normalizeURL(os.Getenv("RSMON_URL")),
|
URL: normalizeURL(os.Getenv("RSMON_URL")),
|
||||||
Token: os.Getenv("RSMON_TOKEN"),
|
Token: os.Getenv("RSMON_TOKEN"),
|
||||||
|
BootstrapToken: os.Getenv("RSMON_BOOTSTRAP_TOKEN"),
|
||||||
|
StateFile: strings.TrimSpace(os.Getenv("RSMON_STATE_FILE")),
|
||||||
|
WorkerID: strings.TrimSpace(os.Getenv("RSMON_WORKER_ID")),
|
||||||
HTTP: HTTPConfigFromEnv(),
|
HTTP: HTTPConfigFromEnv(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -91,7 +142,9 @@ func ConfigFromEnv() Config {
|
|||||||
// HTTPConfigFromEnv reads HTTP listener settings from the environment.
|
// HTTPConfigFromEnv reads HTTP listener settings from the environment.
|
||||||
// Empty WORKER_HOST defaults to DefaultHTTPHost; empty WORKER_PORT defaults
|
// Empty WORKER_HOST defaults to DefaultHTTPHost; empty WORKER_PORT defaults
|
||||||
// to DefaultHTTPPort. A malformed WORKER_PORT falls back to the default.
|
// to DefaultHTTPPort. A malformed WORKER_PORT falls back to the default.
|
||||||
// URL is parsed loosely here; ValidateHTTPConfig does the real check.
|
// The public origin is resolved from PUBLIC_URL (canonical) with the
|
||||||
|
// deprecated WORKER_URL as the migration fallback; it is parsed loosely
|
||||||
|
// here — ValidateHTTPConfig does the real check.
|
||||||
func HTTPConfigFromEnv() HTTPConfig {
|
func HTTPConfigFromEnv() HTTPConfig {
|
||||||
port := DefaultHTTPPort
|
port := DefaultHTTPPort
|
||||||
if raw := strings.TrimSpace(os.Getenv("WORKER_PORT")); raw != "" {
|
if raw := strings.TrimSpace(os.Getenv("WORKER_PORT")); raw != "" {
|
||||||
@@ -103,16 +156,18 @@ func HTTPConfigFromEnv() HTTPConfig {
|
|||||||
if host == "" {
|
if host == "" {
|
||||||
host = DefaultHTTPHost
|
host = DefaultHTTPHost
|
||||||
}
|
}
|
||||||
|
publicURL, source := PublicURLFromEnv()
|
||||||
return HTTPConfig{
|
return HTTPConfig{
|
||||||
Host: host,
|
Host: host,
|
||||||
Port: port,
|
Port: port,
|
||||||
URL: strings.TrimSpace(os.Getenv("WORKER_URL")),
|
PublicURL: publicURL,
|
||||||
|
URLSource: source,
|
||||||
Login: os.Getenv("WORKER_LOGIN"),
|
Login: os.Getenv("WORKER_LOGIN"),
|
||||||
Password: os.Getenv("WORKER_PASSWORD"),
|
Password: os.Getenv("WORKER_PASSWORD"),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// ValidateHTTPConfig enforces the Task 2 invariants:
|
// ValidateHTTPConfig enforces the local HTTP-listener invariants:
|
||||||
//
|
//
|
||||||
// - WORKER_LOGIN and WORKER_PASSWORD must be both set or both empty.
|
// - WORKER_LOGIN and WORKER_PASSWORD must be both set or both empty.
|
||||||
// A mixed state (XOR) is a config bug and must fail fast so an
|
// A mixed state (XOR) is a config bug and must fail fast so an
|
||||||
@@ -121,12 +176,15 @@ func HTTPConfigFromEnv() HTTPConfig {
|
|||||||
// - When the listener would actually start (WORKER_PORT > 0), both
|
// - When the listener would actually start (WORKER_PORT > 0), both
|
||||||
// must be set; otherwise we are going to expose an unauthenticated
|
// must be set; otherwise we are going to expose an unauthenticated
|
||||||
// endpoint.
|
// endpoint.
|
||||||
// - WORKER_URL, if set, must be a parseable absolute URL. Relative
|
// - A configured advertised origin must validate. The canonical
|
||||||
// URLs are rejected because peer workers and the main app need a
|
// PUBLIC_URL is held to the strict origin rules of ValidatePublicURL
|
||||||
// concrete origin to dial.
|
// and, in an explicitly production environment
|
||||||
// - A WORKER_URL with scheme=http on a non-loopback host logs a
|
// (DEPLOY_ENV/RSMON_ENV/GO_ENV = production), plain HTTP on a
|
||||||
// warning: production deployments normally terminate TLS at a
|
// non-loopback host is rejected. A legacy WORKER_URL (URLSource is
|
||||||
// reverse proxy (Traefik, nginx).
|
// PublicURLSourceLegacy) is held only to the tolerant
|
||||||
|
// ValidateAdvertisedURL so shapes that previously ran keep running;
|
||||||
|
// it never fails production startup, preserving the historical
|
||||||
|
// warn-only behavior.
|
||||||
func ValidateHTTPConfig(c HTTPConfig, willListen bool) error {
|
func ValidateHTTPConfig(c HTTPConfig, willListen bool) error {
|
||||||
loginSet := c.Login != ""
|
loginSet := c.Login != ""
|
||||||
passSet := c.Password != ""
|
passSet := c.Password != ""
|
||||||
@@ -138,23 +196,103 @@ func ValidateHTTPConfig(c HTTPConfig, willListen bool) error {
|
|||||||
if willListen && !c.IsAuthConfigured() {
|
if willListen && !c.IsAuthConfigured() {
|
||||||
return fmt.Errorf("HTTP listener refused to start: WORKER_LOGIN and WORKER_PASSWORD must be set when WORKER_PORT > 0")
|
return fmt.Errorf("HTTP listener refused to start: WORKER_LOGIN and WORKER_PASSWORD must be set when WORKER_PORT > 0")
|
||||||
}
|
}
|
||||||
if c.URL == "" {
|
if c.PublicURL == "" {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
u, err := url.Parse(c.URL)
|
if c.URLSource == PublicURLSourceLegacy {
|
||||||
if err != nil {
|
if err := ValidateAdvertisedURL(c.PublicURL); err != nil {
|
||||||
return fmt.Errorf("WORKER_URL is not a valid URL: %v", err)
|
return fmt.Errorf("%s: %w", EnvWorkerURLLegacy, err)
|
||||||
}
|
}
|
||||||
if u.Scheme == "" || u.Host == "" {
|
return nil
|
||||||
return fmt.Errorf("WORKER_URL must be an absolute URL with scheme and host (got %q)", c.URL)
|
}
|
||||||
|
if err := ValidatePublicURL(c.PublicURL); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if productionEnv() {
|
||||||
|
if host, warn := WarnInsecurePublicURL(c.PublicURL); warn {
|
||||||
|
return fmt.Errorf(
|
||||||
|
"%s uses plain HTTP on a non-loopback host (%s), which is not permitted in production; front %s with a TLS-terminating reverse proxy",
|
||||||
|
EnvPublicURL, host, EnvPublicURL,
|
||||||
|
)
|
||||||
}
|
}
|
||||||
if u.Scheme != schemeHTTP && u.Scheme != schemeHTTPS {
|
|
||||||
return fmt.Errorf("WORKER_URL scheme must be http or https (got %q)", u.Scheme)
|
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// WarnInsecurePublicURL logs a warning when WORKER_URL uses plain http
|
// ValidatePublicURL enforces the strict origin shape required by
|
||||||
|
// docs/public-endpoint-and-identity.md ("One Worker, One Public URL")
|
||||||
|
// for the canonical PUBLIC_URL configuration: an absolute http(s) URL
|
||||||
|
// with a scheme and authority and nothing else. Userinfo, query,
|
||||||
|
// fragment, any path other than "/", and a missing hostname (for example
|
||||||
|
// "https://:27401") are rejected because every route lives on the origin
|
||||||
|
// root and credentials must not leak into the advertised endpoint.
|
||||||
|
func ValidatePublicURL(raw string) error {
|
||||||
|
u, err := parseURL(raw, EnvPublicURL)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if u.User != nil {
|
||||||
|
return fmt.Errorf("%s must not contain userinfo (got %q)", EnvPublicURL, raw)
|
||||||
|
}
|
||||||
|
if u.RawQuery != "" {
|
||||||
|
return fmt.Errorf("%s must not contain a query (got %q)", EnvPublicURL, raw)
|
||||||
|
}
|
||||||
|
if u.Fragment != "" {
|
||||||
|
return fmt.Errorf("%s must not contain a fragment (got %q)", EnvPublicURL, raw)
|
||||||
|
}
|
||||||
|
if u.Path != "" && u.Path != "/" {
|
||||||
|
return fmt.Errorf("%s must not contain a path (got %q)", EnvPublicURL, raw)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// ValidateAdvertisedURL enforces the tolerant absolute-URL shape used
|
||||||
|
// for the legacy WORKER_URL and for endpoints supplied by the control
|
||||||
|
// plane: an http(s) URL with a scheme, a host, and a non-empty hostname.
|
||||||
|
// Unlike ValidatePublicURL it does not reject a path, userinfo, query,
|
||||||
|
// or fragment, because legacy configurations and old control planes may
|
||||||
|
// carry such shapes and must not newly fail. It still rejects values
|
||||||
|
// that could never be dialed, such as "https://:27401".
|
||||||
|
func ValidateAdvertisedURL(raw string) error {
|
||||||
|
_, err := parseURL(raw, EnvWorkerURLLegacy)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// parseURL parses an absolute http(s) advertised URL. Both validators
|
||||||
|
// share the scheme/host/hostname rules; label names the source in error
|
||||||
|
// messages so diagnostics point at the variable the operator set.
|
||||||
|
func parseURL(raw, label string) (*url.URL, error) {
|
||||||
|
u, err := url.Parse(raw)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("%s is not a valid URL: %v", label, err)
|
||||||
|
}
|
||||||
|
if u.Scheme == "" || u.Host == "" {
|
||||||
|
return nil, fmt.Errorf("%s must be an absolute URL with scheme and host (got %q)", label, raw)
|
||||||
|
}
|
||||||
|
if u.Scheme != schemeHTTP && u.Scheme != schemeHTTPS {
|
||||||
|
return nil, fmt.Errorf("%s scheme must be http or https (got %q)", label, u.Scheme)
|
||||||
|
}
|
||||||
|
if u.Hostname() == "" {
|
||||||
|
return nil, fmt.Errorf("%s must include a host (got %q)", label, raw)
|
||||||
|
}
|
||||||
|
return u, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// productionEnv reports whether the worker is explicitly configured for
|
||||||
|
// production. Only an explicit literal enables the HTTPS-only PUBLIC_URL
|
||||||
|
// policy; an unset variable keeps the historical warn-on-plain-HTTP
|
||||||
|
// behavior so existing installations upgrade without surprise.
|
||||||
|
func productionEnv() bool {
|
||||||
|
for _, key := range []string{"DEPLOY_ENV", "RSMON_ENV", "GO_ENV"} {
|
||||||
|
switch strings.ToLower(strings.TrimSpace(os.Getenv(key))) {
|
||||||
|
case "production", "prod":
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// WarnInsecurePublicURL logs a warning when PUBLIC_URL uses plain http
|
||||||
// for a non-loopback host. Returns the host so the caller can log it.
|
// for a non-loopback host. Returns the host so the caller can log it.
|
||||||
// A no-op for the loopback case (typical local dev) and for https URLs.
|
// A no-op for the loopback case (typical local dev) and for https URLs.
|
||||||
func WarnInsecurePublicURL(rawURL string) (host string, shouldWarn bool) {
|
func WarnInsecurePublicURL(rawURL string) (host string, shouldWarn bool) {
|
||||||
|
|||||||
@@ -9,12 +9,13 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
// TestHTTPConfigFromEnv_Defaults verifies the documented defaults when no
|
// TestHTTPConfigFromEnv_Defaults verifies the documented defaults when no
|
||||||
// HTTP-related env vars are set: 0.0.0.0:27401 and empty URL / login /
|
// HTTP-related env vars are set: 0.0.0.0:27401 and empty public URL /
|
||||||
// password. The default port was bumped from 7401 to 27401 to avoid
|
// login / password. The default port was bumped from 7401 to 27401 to avoid
|
||||||
// colliding with the main RSMon app when the worker is co-located.
|
// colliding with the main RSMon app when the worker is co-located.
|
||||||
func TestHTTPConfigFromEnv_Defaults(t *testing.T) {
|
func TestHTTPConfigFromEnv_Defaults(t *testing.T) {
|
||||||
t.Setenv("WORKER_HOST", "")
|
t.Setenv("WORKER_HOST", "")
|
||||||
t.Setenv("WORKER_PORT", "")
|
t.Setenv("WORKER_PORT", "")
|
||||||
|
t.Setenv("PUBLIC_URL", "")
|
||||||
t.Setenv("WORKER_URL", "")
|
t.Setenv("WORKER_URL", "")
|
||||||
t.Setenv("WORKER_LOGIN", "")
|
t.Setenv("WORKER_LOGIN", "")
|
||||||
t.Setenv("WORKER_PASSWORD", "")
|
t.Setenv("WORKER_PASSWORD", "")
|
||||||
@@ -22,7 +23,7 @@ func TestHTTPConfigFromEnv_Defaults(t *testing.T) {
|
|||||||
cfg := HTTPConfigFromEnv()
|
cfg := HTTPConfigFromEnv()
|
||||||
assert.Equal(t, DefaultHTTPHost, cfg.Host, "host should default to 0.0.0.0")
|
assert.Equal(t, DefaultHTTPHost, cfg.Host, "host should default to 0.0.0.0")
|
||||||
assert.Equal(t, DefaultHTTPPort, cfg.Port, "port should default to 27401")
|
assert.Equal(t, DefaultHTTPPort, cfg.Port, "port should default to 27401")
|
||||||
assert.Equal(t, "", cfg.URL)
|
assert.Equal(t, "", cfg.PublicURL)
|
||||||
assert.Equal(t, "", cfg.Login)
|
assert.Equal(t, "", cfg.Login)
|
||||||
assert.Equal(t, "", cfg.Password)
|
assert.Equal(t, "", cfg.Password)
|
||||||
assert.False(t, cfg.IsAuthConfigured())
|
assert.False(t, cfg.IsAuthConfigured())
|
||||||
@@ -33,14 +34,14 @@ func TestHTTPConfigFromEnv_Defaults(t *testing.T) {
|
|||||||
func TestHTTPConfigFromEnv_Overrides(t *testing.T) {
|
func TestHTTPConfigFromEnv_Overrides(t *testing.T) {
|
||||||
t.Setenv("WORKER_HOST", "127.0.0.1")
|
t.Setenv("WORKER_HOST", "127.0.0.1")
|
||||||
t.Setenv("WORKER_PORT", "9100")
|
t.Setenv("WORKER_PORT", "9100")
|
||||||
t.Setenv("WORKER_URL", "https://worker.example.com")
|
t.Setenv("PUBLIC_URL", "https://worker.example.com")
|
||||||
t.Setenv("WORKER_LOGIN", "ops")
|
t.Setenv("WORKER_LOGIN", "ops")
|
||||||
t.Setenv("WORKER_PASSWORD", "s3cret")
|
t.Setenv("WORKER_PASSWORD", "s3cret")
|
||||||
|
|
||||||
cfg := HTTPConfigFromEnv()
|
cfg := HTTPConfigFromEnv()
|
||||||
assert.Equal(t, "127.0.0.1", cfg.Host)
|
assert.Equal(t, "127.0.0.1", cfg.Host)
|
||||||
assert.Equal(t, 9100, cfg.Port)
|
assert.Equal(t, 9100, cfg.Port)
|
||||||
assert.Equal(t, "https://worker.example.com", cfg.URL)
|
assert.Equal(t, "https://worker.example.com", cfg.PublicURL)
|
||||||
assert.Equal(t, "ops", cfg.Login)
|
assert.Equal(t, "ops", cfg.Login)
|
||||||
assert.Equal(t, "s3cret", cfg.Password)
|
assert.Equal(t, "s3cret", cfg.Password)
|
||||||
assert.True(t, cfg.IsAuthConfigured())
|
assert.True(t, cfg.IsAuthConfigured())
|
||||||
@@ -70,6 +71,44 @@ func TestHTTPConfigFromEnv_PortOutOfRangeFallsBackToDefault(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestPublicURLFromEnv_PublicURLWins pins the canonical-source rule:
|
||||||
|
// when both PUBLIC_URL and the legacy WORKER_URL are set, PUBLIC_URL wins
|
||||||
|
// and the source is reported as canonical.
|
||||||
|
func TestPublicURLFromEnv_PublicURLWins(t *testing.T) {
|
||||||
|
t.Setenv("PUBLIC_URL", "https://canonical.example.com")
|
||||||
|
t.Setenv("WORKER_URL", "http://legacy.example.com:27401")
|
||||||
|
|
||||||
|
url, source := PublicURLFromEnv()
|
||||||
|
assert.Equal(t, "https://canonical.example.com", url)
|
||||||
|
assert.Equal(t, PublicURLSourceCanonical, source)
|
||||||
|
|
||||||
|
cfg := HTTPConfigFromEnv()
|
||||||
|
assert.Equal(t, "https://canonical.example.com", cfg.PublicURL)
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestPublicURLFromEnv_LegacyFallback verifies the bounded migration: the
|
||||||
|
// legacy WORKER_URL is resolved when PUBLIC_URL is unset and the source is
|
||||||
|
// reported as legacy so the caller can emit the deprecation warning.
|
||||||
|
func TestPublicURLFromEnv_LegacyFallback(t *testing.T) {
|
||||||
|
t.Setenv("PUBLIC_URL", "")
|
||||||
|
t.Setenv("WORKER_URL", "https://legacy.example.com")
|
||||||
|
|
||||||
|
url, source := PublicURLFromEnv()
|
||||||
|
assert.Equal(t, "https://legacy.example.com", url)
|
||||||
|
assert.Equal(t, PublicURLSourceLegacy, source)
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestPublicURLFromEnv_None verifies that no configured origin reports the
|
||||||
|
// none source and an empty value.
|
||||||
|
func TestPublicURLFromEnv_None(t *testing.T) {
|
||||||
|
t.Setenv("PUBLIC_URL", "")
|
||||||
|
t.Setenv("WORKER_URL", "")
|
||||||
|
|
||||||
|
url, source := PublicURLFromEnv()
|
||||||
|
assert.Equal(t, "", url)
|
||||||
|
assert.Equal(t, PublicURLSourceNone, source)
|
||||||
|
}
|
||||||
|
|
||||||
// TestValidateHTTPConfig_LoginXORPasswordRejected covers the central
|
// TestValidateHTTPConfig_LoginXORPasswordRejected covers the central
|
||||||
// invariant: a mixed login/password state is a config bug and must
|
// invariant: a mixed login/password state is a config bug and must
|
||||||
// fail fast.
|
// fail fast.
|
||||||
@@ -130,7 +169,7 @@ func TestValidateHTTPConfig_URLAbsoluteRequired(t *testing.T) {
|
|||||||
}
|
}
|
||||||
for _, tc := range cases {
|
for _, tc := range cases {
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
cfg := HTTPConfig{Host: "0.0.0.0", Port: 27401, URL: tc.url}
|
cfg := HTTPConfig{Host: "0.0.0.0", Port: 27401, PublicURL: tc.url}
|
||||||
err := ValidateHTTPConfig(cfg, false)
|
err := ValidateHTTPConfig(cfg, false)
|
||||||
// empty url is allowed
|
// empty url is allowed
|
||||||
if tc.url == "" {
|
if tc.url == "" {
|
||||||
@@ -148,7 +187,7 @@ func TestValidateHTTPConfig_URLSchemeAllowed(t *testing.T) {
|
|||||||
t.Run(scheme, func(t *testing.T) {
|
t.Run(scheme, func(t *testing.T) {
|
||||||
cfg := HTTPConfig{
|
cfg := HTTPConfig{
|
||||||
Host: "0.0.0.0", Port: 27401,
|
Host: "0.0.0.0", Port: 27401,
|
||||||
URL: scheme + "://localhost:27401",
|
PublicURL: scheme + "://localhost:27401",
|
||||||
}
|
}
|
||||||
assert.NoError(t, ValidateHTTPConfig(cfg, false))
|
assert.NoError(t, ValidateHTTPConfig(cfg, false))
|
||||||
})
|
})
|
||||||
@@ -161,7 +200,7 @@ func TestValidateHTTPConfig_URLSchemeRejected(t *testing.T) {
|
|||||||
t.Run(scheme, func(t *testing.T) {
|
t.Run(scheme, func(t *testing.T) {
|
||||||
cfg := HTTPConfig{
|
cfg := HTTPConfig{
|
||||||
Host: "0.0.0.0", Port: 27401,
|
Host: "0.0.0.0", Port: 27401,
|
||||||
URL: scheme + "://localhost:27401",
|
PublicURL: scheme + "://localhost:27401",
|
||||||
}
|
}
|
||||||
err := ValidateHTTPConfig(cfg, false)
|
err := ValidateHTTPConfig(cfg, false)
|
||||||
require.Error(t, err)
|
require.Error(t, err)
|
||||||
@@ -170,6 +209,151 @@ func TestValidateHTTPConfig_URLSchemeRejected(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestValidatePublicURL_OriginShapeRejected covers the plan's negative
|
||||||
|
// cases: the advertised origin must be scheme + authority only, with no
|
||||||
|
// userinfo, query, fragment, or ambiguous path.
|
||||||
|
func TestValidatePublicURL_OriginShapeRejected(t *testing.T) {
|
||||||
|
cases := []struct {
|
||||||
|
name string
|
||||||
|
url string
|
||||||
|
want string
|
||||||
|
}{
|
||||||
|
{"userinfo", "https://user:pass@worker.example.com", "userinfo"},
|
||||||
|
{"query", "https://worker.example.com?x=1", "query"},
|
||||||
|
{"fragment", "https://worker.example.com#frag", "fragment"},
|
||||||
|
{"path", "https://worker.example.com/web", "path"},
|
||||||
|
{"path nested", "https://worker.example.com/raft/", "path"},
|
||||||
|
{"missing scheme", "worker.example.com", "absolute url"},
|
||||||
|
{"missing host", "https://", "absolute url"},
|
||||||
|
{"bad scheme", "ftp://worker.example.com", "scheme must be http or https"},
|
||||||
|
}
|
||||||
|
for _, tc := range cases {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
err := ValidatePublicURL(tc.url)
|
||||||
|
require.Error(t, err)
|
||||||
|
assert.Contains(t, strings.ToLower(err.Error()), tc.want)
|
||||||
|
assert.Contains(t, err.Error(), EnvPublicURL,
|
||||||
|
"error must name the canonical PUBLIC_URL variable")
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestValidatePublicURL_RootSlashAllowed confirms that an empty path and
|
||||||
|
// the root path "/" are both accepted as an origin.
|
||||||
|
func TestValidatePublicURL_RootSlashAllowed(t *testing.T) {
|
||||||
|
for _, u := range []string{"https://worker.example.com", "https://worker.example.com/"} {
|
||||||
|
assert.NoError(t, ValidatePublicURL(u))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestValidatePublicURL_HostnameRequired pins the hostname rule: an
|
||||||
|
// authority that parses to no hostname (for example "https://:27401")
|
||||||
|
// must be rejected even though it has a host:port string.
|
||||||
|
func TestValidatePublicURL_HostnameRequired(t *testing.T) {
|
||||||
|
for _, u := range []string{"https://:27401", "http://:7401"} {
|
||||||
|
t.Run(u, func(t *testing.T) {
|
||||||
|
err := ValidatePublicURL(u)
|
||||||
|
require.Error(t, err)
|
||||||
|
assert.Contains(t, strings.ToLower(err.Error()), "must include a host")
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestValidateAdvertisedURL_Tolerant pins the legacy-tolerant shape used
|
||||||
|
// for WORKER_URL and control-plane-supplied endpoints: an absolute
|
||||||
|
// http(s) URL with a scheme, host, and hostname. Paths, userinfo, query,
|
||||||
|
// and fragments that previously ran must keep validating; only values
|
||||||
|
// that could never be dialed (relative, bad scheme, missing hostname)
|
||||||
|
// are rejected.
|
||||||
|
func TestValidateAdvertisedURL_Tolerant(t *testing.T) {
|
||||||
|
for _, u := range []string{
|
||||||
|
"https://worker.example.com",
|
||||||
|
"https://worker.example.com/web",
|
||||||
|
"https://worker.example.com/raft/",
|
||||||
|
"http://localhost:27401",
|
||||||
|
"https://user:pass@worker.example.com/web?x=1#frag",
|
||||||
|
} {
|
||||||
|
assert.NoError(t, ValidateAdvertisedURL(u), "tolerant validator must accept %q", u)
|
||||||
|
}
|
||||||
|
for _, u := range []string{
|
||||||
|
"worker.example.com",
|
||||||
|
"https://",
|
||||||
|
"https://:27401",
|
||||||
|
"ftp://worker.example.com",
|
||||||
|
"file:///tmp/x",
|
||||||
|
} {
|
||||||
|
assert.Error(t, ValidateAdvertisedURL(u), "tolerant validator must reject %q", u)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestValidateHTTPConfig_LegacySourceTolerant is the core bounded-migration
|
||||||
|
// guarantee: a legacy WORKER_URL that previously ran (path, userinfo, even
|
||||||
|
// plain HTTP in production) must not newly fail startup. Only genuinely
|
||||||
|
// unusable values are rejected.
|
||||||
|
func TestValidateHTTPConfig_LegacySourceTolerant(t *testing.T) {
|
||||||
|
t.Setenv("DEPLOY_ENV", "production")
|
||||||
|
|
||||||
|
for _, u := range []string{
|
||||||
|
"https://worker.example.com",
|
||||||
|
"https://worker.example.com/web",
|
||||||
|
"http://worker.example.com",
|
||||||
|
} {
|
||||||
|
cfg := HTTPConfig{Host: "0.0.0.0", Port: 27401, PublicURL: u, URLSource: PublicURLSourceLegacy}
|
||||||
|
assert.NoError(t, ValidateHTTPConfig(cfg, false),
|
||||||
|
"legacy WORKER_URL=%q must not newly fail startup", u)
|
||||||
|
}
|
||||||
|
|
||||||
|
cfg := HTTPConfig{Host: "0.0.0.0", Port: 27401, PublicURL: "https://:27401", URLSource: PublicURLSourceLegacy}
|
||||||
|
err := ValidateHTTPConfig(cfg, false)
|
||||||
|
require.Error(t, err)
|
||||||
|
assert.Contains(t, err.Error(), EnvWorkerURLLegacy,
|
||||||
|
"legacy diagnostics must name the WORKER_URL source")
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestValidateHTTPConfig_CanonicalSourceStrict verifies the canonical
|
||||||
|
// PUBLIC_URL stays strict even when the same value would be tolerated as
|
||||||
|
// a legacy WORKER_URL.
|
||||||
|
func TestValidateHTTPConfig_CanonicalSourceStrict(t *testing.T) {
|
||||||
|
cfg := HTTPConfig{Host: "0.0.0.0", Port: 27401, PublicURL: "https://worker.example.com/web", URLSource: PublicURLSourceCanonical}
|
||||||
|
err := ValidateHTTPConfig(cfg, false)
|
||||||
|
require.Error(t, err)
|
||||||
|
assert.Contains(t, err.Error(), EnvPublicURL,
|
||||||
|
"canonical diagnostics must name the PUBLIC_URL source")
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestValidateHTTPConfig_ProductionRejectsPlainHTTP pins the production
|
||||||
|
// HTTPS policy for the canonical PUBLIC_URL: an explicitly production
|
||||||
|
// worker must not advertise plain HTTP on a non-loopback host. Loopback
|
||||||
|
// and https stay accepted.
|
||||||
|
func TestValidateHTTPConfig_ProductionRejectsPlainHTTP(t *testing.T) {
|
||||||
|
t.Setenv("DEPLOY_ENV", "production")
|
||||||
|
|
||||||
|
err := ValidateHTTPConfig(HTTPConfig{
|
||||||
|
Host: "0.0.0.0", Port: 27401,
|
||||||
|
PublicURL: "http://worker.example.com",
|
||||||
|
URLSource: PublicURLSourceCanonical,
|
||||||
|
}, false)
|
||||||
|
require.Error(t, err)
|
||||||
|
assert.Contains(t, err.Error(), "not permitted in production")
|
||||||
|
|
||||||
|
cfg := HTTPConfig{Host: "0.0.0.0", Port: 27401, PublicURL: "https://worker.example.com", URLSource: PublicURLSourceCanonical}
|
||||||
|
assert.NoError(t, ValidateHTTPConfig(cfg, false))
|
||||||
|
|
||||||
|
cfg = HTTPConfig{Host: "0.0.0.0", Port: 27401, PublicURL: "http://localhost:27401", URLSource: PublicURLSourceCanonical}
|
||||||
|
assert.NoError(t, ValidateHTTPConfig(cfg, false))
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestValidateHTTPConfig_NonProductionAllowsPlainHTTP keeps the historical
|
||||||
|
// warn-only behavior when no explicit production environment is configured.
|
||||||
|
func TestValidateHTTPConfig_NonProductionAllowsPlainHTTP(t *testing.T) {
|
||||||
|
t.Setenv("DEPLOY_ENV", "")
|
||||||
|
t.Setenv("RSMON_ENV", "development")
|
||||||
|
t.Setenv("GO_ENV", "")
|
||||||
|
|
||||||
|
cfg := HTTPConfig{Host: "0.0.0.0", Port: 27401, PublicURL: "http://worker.example.com"}
|
||||||
|
assert.NoError(t, ValidateHTTPConfig(cfg, false))
|
||||||
|
}
|
||||||
|
|
||||||
// TestWarnInsecurePublicURL exercises the warning helper: only http on
|
// TestWarnInsecurePublicURL exercises the warning helper: only http on
|
||||||
// non-loopback hosts should warn. The returned host is the parsed host
|
// non-loopback hosts should warn. The returned host is the parsed host
|
||||||
// (host:port when present) so the caller can log a useful target.
|
// (host:port when present) so the caller can log a useful target.
|
||||||
|
|||||||
@@ -25,14 +25,27 @@ import (
|
|||||||
// silently dropping it as the production plan forbids).
|
// silently dropping it as the production plan forbids).
|
||||||
//
|
//
|
||||||
//nolint:gocritic // task model is shared with the rest of the dispatcher; keep by-value
|
//nolint:gocritic // task model is shared with the rest of the dispatcher; keep by-value
|
||||||
func (r *Runner) ExecuteNotification(ctx context.Context, task models.Task) wire.NotificationResultReport {
|
func (r *Runner) ExecuteNotification(ctx context.Context, task models.Task) (report wire.NotificationResultReport) {
|
||||||
start := time.Now()
|
start := time.Now()
|
||||||
|
method := ""
|
||||||
report := wire.NotificationResultReport{
|
report = wire.NotificationResultReport{
|
||||||
JobID: task.JobID,
|
JobID: task.JobID,
|
||||||
Status: wire.NotificationResultPermanent,
|
Status: wire.NotificationResultPermanent,
|
||||||
DurationMs: 0,
|
DurationMs: 0,
|
||||||
}
|
}
|
||||||
|
defer func() {
|
||||||
|
if recover() != nil {
|
||||||
|
report.Status = wire.NotificationResultPermanent
|
||||||
|
report.ProviderResponse = nil
|
||||||
|
report.RetryAfterSeconds = nil
|
||||||
|
report.Error = stringPtr("notification executor panic")
|
||||||
|
}
|
||||||
|
report.DurationMs = int(time.Since(start) / time.Millisecond)
|
||||||
|
r.recordDelegatedNotification(report.JobID, method, report)
|
||||||
|
}()
|
||||||
|
if r.notificationExecutor != nil {
|
||||||
|
return r.notificationExecutor(ctx, task)
|
||||||
|
}
|
||||||
|
|
||||||
if len(task.Payload) == 0 {
|
if len(task.Payload) == 0 {
|
||||||
report.Status = wire.NotificationResultPermanent
|
report.Status = wire.NotificationResultPermanent
|
||||||
@@ -50,6 +63,7 @@ func (r *Runner) ExecuteNotification(ctx context.Context, task models.Task) wire
|
|||||||
if nt.JobID == "" {
|
if nt.JobID == "" {
|
||||||
nt.JobID = task.JobID
|
nt.JobID = task.JobID
|
||||||
}
|
}
|
||||||
|
method = nt.Method
|
||||||
if nt.MessageID == 0 && task.MessageID != nil {
|
if nt.MessageID == 0 && task.MessageID != nil {
|
||||||
nt.MessageID = *task.MessageID
|
nt.MessageID = *task.MessageID
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -2,7 +2,10 @@ package distworker
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"encoding/json"
|
||||||
"errors"
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"sync"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
@@ -228,6 +231,177 @@ func TestExecuteNotification_ReportsJobIDFromPayload(t *testing.T) {
|
|||||||
assert.Equal(t, "outer-job", report.JobID)
|
assert.Equal(t, "outer-job", report.JobID)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestDelegatedNotificationAuditCoexistsWithSelfcheckWithoutSecrets(t *testing.T) {
|
||||||
|
const (
|
||||||
|
recipient = "ops-private@example.com"
|
||||||
|
renderedBody = "private rendered body"
|
||||||
|
credentialSecret = "smtp-password-secret"
|
||||||
|
hookURL = "https://hooks.example.com/private"
|
||||||
|
authorization = "Bearer private-authorization"
|
||||||
|
providerResponse = "private provider response"
|
||||||
|
)
|
||||||
|
r := runnerWithCreds(&wire.NotificationCredentials{
|
||||||
|
SMTP: []wire.SMTPCredential{{Password: credentialSecret}},
|
||||||
|
})
|
||||||
|
r.notifyResults = make(chan notifyResultEnvelope, 1)
|
||||||
|
r.RecordNotification(&NotificationRow{
|
||||||
|
Kind: "email", Channel: notificationChannelSMTP, Subject: "selfcheck", Body: "main API down", OK: true,
|
||||||
|
})
|
||||||
|
r.executeAndForwardNotification(wire.NotificationTask{
|
||||||
|
JobID: "delegated-job", Method: "sms", Subject: authorization,
|
||||||
|
BodyText: renderedBody, BodyHTML: providerResponse,
|
||||||
|
Contact: wire.NotificationContact{Kind: "sms", Value: recipient, Name: hookURL},
|
||||||
|
})
|
||||||
|
|
||||||
|
rows := r.RecentNotifications(10)
|
||||||
|
require.Len(t, rows, 2)
|
||||||
|
assert.Equal(t, "selfcheck", rows[0].Subject)
|
||||||
|
delegated := rows[1]
|
||||||
|
assert.Equal(t, "delegated-job", delegated.JobID)
|
||||||
|
assert.Equal(t, "sms", delegated.Method)
|
||||||
|
assert.Equal(t, wire.NotificationResultPermanent, delegated.Status)
|
||||||
|
assert.Empty(t, delegated.Subject)
|
||||||
|
assert.Empty(t, delegated.Body)
|
||||||
|
assert.Empty(t, delegated.Error)
|
||||||
|
assert.False(t, delegated.At.IsZero())
|
||||||
|
|
||||||
|
stored, err := json.Marshal(delegated)
|
||||||
|
require.NoError(t, err)
|
||||||
|
for _, secret := range []string{recipient, renderedBody, credentialSecret, hookURL, authorization, providerResponse} {
|
||||||
|
assert.NotContains(t, string(stored), secret)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDelegatedNotificationAuditRecordsOneTerminalRow(t *testing.T) {
|
||||||
|
const secret = "private-notification-secret"
|
||||||
|
validTask := func(jobID string) wire.NotificationTask {
|
||||||
|
return wire.NotificationTask{
|
||||||
|
JobID: jobID, LeaseToken: "lease", Method: "sms", Subject: secret, BodyText: secret,
|
||||||
|
Contact: wire.NotificationContact{Value: secret, Name: secret},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
invoke func(*Runner)
|
||||||
|
method string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "normal result",
|
||||||
|
invoke: func(r *Runner) { r.executeAndForwardNotification(validTask("normal")) },
|
||||||
|
method: "sms",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "invalid deadline",
|
||||||
|
invoke: func(r *Runner) {
|
||||||
|
task := validTask("invalid-deadline")
|
||||||
|
deadline := "not-a-timestamp"
|
||||||
|
task.Deadline = &deadline
|
||||||
|
r.executeAndForwardNotification(task)
|
||||||
|
},
|
||||||
|
method: "sms",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "expired deadline",
|
||||||
|
invoke: func(r *Runner) {
|
||||||
|
task := validTask("expired-deadline")
|
||||||
|
deadline := time.Now().Add(-time.Second).Format(time.RFC3339Nano)
|
||||||
|
task.Deadline = &deadline
|
||||||
|
r.executeAndForwardNotification(task)
|
||||||
|
},
|
||||||
|
method: "sms",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "malformed payload",
|
||||||
|
invoke: func(r *Runner) {
|
||||||
|
r.ExecuteNotification(context.Background(), models.Task{JobID: "malformed-payload", Payload: []byte(`{"subject":"` + secret)})
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "executor panic",
|
||||||
|
invoke: func(r *Runner) {
|
||||||
|
r.notificationExecutor = func(context.Context, models.Task) wire.NotificationResultReport {
|
||||||
|
panic(secret)
|
||||||
|
}
|
||||||
|
r.ExecuteNotification(context.Background(), models.Task{JobID: "panic", Payload: []byte(`{}`)})
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "malformed envelope",
|
||||||
|
invoke: func(r *Runner) {
|
||||||
|
r.enqueueTaskMessage(wire.WorkerMessage{TaskEnvelope: &wire.TaskEnvelope{
|
||||||
|
Type: "invalid", JobID: "bad-envelope", Notify: ptrNotificationTask(validTask("bad-envelope")),
|
||||||
|
}})
|
||||||
|
},
|
||||||
|
method: "sms",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
for _, tc := range tests {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
r := runnerWithCreds(&wire.NotificationCredentials{})
|
||||||
|
r.notifyResults = make(chan notifyResultEnvelope, 1)
|
||||||
|
tc.invoke(r)
|
||||||
|
|
||||||
|
rows := r.RecentNotifications(10)
|
||||||
|
require.Len(t, rows, 1)
|
||||||
|
assert.Equal(t, wire.NotificationResultPermanent, rows[0].Status)
|
||||||
|
assert.Equal(t, tc.method, rows[0].Method)
|
||||||
|
stored, err := json.Marshal(rows[0])
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.NotContains(t, string(stored), secret)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDelegatedNotificationAuditConcurrentRowsAreSecretFree(t *testing.T) {
|
||||||
|
const attempts = 32
|
||||||
|
const secret = "concurrent-private-secret"
|
||||||
|
r := runnerWithCreds(&wire.NotificationCredentials{})
|
||||||
|
var wg sync.WaitGroup
|
||||||
|
for i := 0; i < attempts; i++ {
|
||||||
|
wg.Add(1)
|
||||||
|
go func(i int) {
|
||||||
|
defer wg.Done()
|
||||||
|
r.ExecuteNotification(context.Background(), models.Task{
|
||||||
|
JobID: fmt.Sprintf("job-%d", i),
|
||||||
|
Payload: []byte(`{"job_id":"job-` + fmt.Sprint(i) + `","method":"sms","body_text":"` + secret + `","contact":{"value":"` + secret + `"}}`),
|
||||||
|
})
|
||||||
|
}(i)
|
||||||
|
}
|
||||||
|
wg.Wait()
|
||||||
|
|
||||||
|
rows := r.RecentNotifications(attempts)
|
||||||
|
require.Len(t, rows, attempts)
|
||||||
|
for _, row := range rows {
|
||||||
|
stored, err := json.Marshal(row)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.NotContains(t, string(stored), secret)
|
||||||
|
assert.Equal(t, wire.NotificationResultPermanent, row.Status)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestInvalidNotificationDeadlineIsNotExecuted(t *testing.T) {
|
||||||
|
r := runnerWithCreds(&wire.NotificationCredentials{})
|
||||||
|
r.notifyResults = make(chan notifyResultEnvelope, 1)
|
||||||
|
called := false
|
||||||
|
r.notificationExecutor = func(context.Context, models.Task) wire.NotificationResultReport {
|
||||||
|
called = true
|
||||||
|
return wire.NotificationResultReport{Status: wire.NotificationResultDelivered}
|
||||||
|
}
|
||||||
|
deadline := "not-a-timestamp"
|
||||||
|
r.executeAndForwardNotification(wire.NotificationTask{
|
||||||
|
JobID: "invalid-deadline", LeaseToken: "lease", Method: "sms", Deadline: &deadline,
|
||||||
|
})
|
||||||
|
|
||||||
|
assert.False(t, called)
|
||||||
|
env := <-r.notifyResults
|
||||||
|
assert.Equal(t, wire.NotificationResultPermanent, env.report.Status)
|
||||||
|
require.NotNil(t, env.report.Error)
|
||||||
|
assert.Equal(t, "invalid notification deadline", *env.report.Error)
|
||||||
|
require.Len(t, r.RecentNotifications(10), 1)
|
||||||
|
}
|
||||||
|
|
||||||
|
func ptrNotificationTask(task wire.NotificationTask) *wire.NotificationTask { return &task }
|
||||||
|
|
||||||
// guard against time import being unused if the above compile-time helpers
|
// guard against time import being unused if the above compile-time helpers
|
||||||
// are dropped in a future refactor.
|
// are dropped in a future refactor.
|
||||||
var _ = time.Second
|
var _ = time.Second
|
||||||
|
|||||||
@@ -14,10 +14,7 @@ import (
|
|||||||
const recentResultsSize = 200
|
const recentResultsSize = 200
|
||||||
|
|
||||||
// recentNotificationsSize mirrors recentResultsSize for emitted
|
// recentNotificationsSize mirrors recentResultsSize for emitted
|
||||||
// notifications. Phase 1 only writes selfcheck alerts to this
|
// notifications.
|
||||||
// buffer (the main app's notification flow still lives in the main
|
|
||||||
// app); the buffer is shape-stable so future phases can append
|
|
||||||
// without changing the page contract.
|
|
||||||
const recentNotificationsSize = 100
|
const recentNotificationsSize = 100
|
||||||
|
|
||||||
// ResultRow is one row from the worker's in-memory result ring
|
// ResultRow is one row from the worker's in-memory result ring
|
||||||
@@ -35,9 +32,9 @@ type ResultRow struct {
|
|||||||
At time.Time
|
At time.Time
|
||||||
}
|
}
|
||||||
|
|
||||||
// NotificationRow is one row from the worker's notification ring
|
// NotificationRow is one row from the worker's notification ring buffer.
|
||||||
// buffer. Phase 1 only fills this from selfcheck alerts; the row
|
// Delegated rows use only JobID, Method, Status, DurationMs, and At. They
|
||||||
// shape is forward-compatible with main-app-issued notifications.
|
// deliberately omit delivery inputs and provider output.
|
||||||
type NotificationRow struct {
|
type NotificationRow struct {
|
||||||
Kind string // "email", "telegram_private", "telegram_group"
|
Kind string // "email", "telegram_private", "telegram_group"
|
||||||
Channel string
|
Channel string
|
||||||
@@ -46,6 +43,11 @@ type NotificationRow struct {
|
|||||||
OK bool
|
OK bool
|
||||||
Error string
|
Error string
|
||||||
At time.Time
|
At time.Time
|
||||||
|
|
||||||
|
JobID string
|
||||||
|
Method string
|
||||||
|
Status string
|
||||||
|
DurationMs int
|
||||||
}
|
}
|
||||||
|
|
||||||
// resultBuffer is a thread-safe FIFO ring buffer of ResultRow. The
|
// resultBuffer is a thread-safe FIFO ring buffer of ResultRow. The
|
||||||
|
|||||||
Разница между файлами не показана из-за своего большого размера
Загрузить разницу
@@ -1,8 +1,13 @@
|
|||||||
package distworker
|
package distworker
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"sync"
|
||||||
"testing"
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/gorilla/websocket"
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
|
|
||||||
@@ -11,9 +16,208 @@ import (
|
|||||||
|
|
||||||
func TestEnqueueTaskMessagePrefersEnvelopeOverLegacyCheck(t *testing.T) {
|
func TestEnqueueTaskMessagePrefersEnvelopeOverLegacyCheck(t *testing.T) {
|
||||||
r := &Runner{jobQueue: make(chan wire.CheckJob, 2), stopCh: make(chan struct{})}
|
r := &Runner{jobQueue: make(chan wire.CheckJob, 2), stopCh: make(chan struct{})}
|
||||||
message := wire.WorkerMessage{Kind: "task", TaskEnvelope: &wire.TaskEnvelope{Type: wire.TaskTypeCheck, Job: &wire.CheckJob{JobID: "v2"}}, Task: &wire.CheckJob{JobID: "v1"}}
|
message := wire.WorkerMessage{Kind: "task", TaskEnvelope: &wire.TaskEnvelope{Type: wire.TaskTypeCheck, JobID: "v2", Job: &wire.CheckJob{JobID: "v2", LeaseToken: "lease-2", Kind: "http"}}, Task: &wire.CheckJob{JobID: "v1"}}
|
||||||
require.True(t, r.enqueueTaskMessage(message))
|
require.True(t, r.enqueueTaskMessage(message))
|
||||||
job := <-r.jobQueue
|
job := <-r.jobQueue
|
||||||
assert.Equal(t, "v2", job.JobID)
|
assert.Equal(t, "v2", job.JobID)
|
||||||
assert.Empty(t, r.jobQueue, "the sibling legacy field must not run a second check")
|
assert.Empty(t, r.jobQueue, "the sibling legacy field must not run a second check")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestDrainFinalResultsBoundsAndReportsStaleLeases(t *testing.T) {
|
||||||
|
upgrader := websocket.Upgrader{}
|
||||||
|
reports := make(chan wire.WorkerMessage, finalDrainLimit+2)
|
||||||
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
|
||||||
|
conn, err := upgrader.Upgrade(w, req, nil)
|
||||||
|
require.NoError(t, err)
|
||||||
|
defer conn.Close()
|
||||||
|
for {
|
||||||
|
var message wire.WorkerMessage
|
||||||
|
if err := conn.ReadJSON(&message); err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
reports <- message
|
||||||
|
}
|
||||||
|
}))
|
||||||
|
defer server.Close()
|
||||||
|
wsURL := "ws" + server.URL[len("http"):]
|
||||||
|
conn, _, err := websocket.DefaultDialer.Dial(wsURL, nil)
|
||||||
|
require.NoError(t, err)
|
||||||
|
defer conn.Close()
|
||||||
|
r := &Runner{controlConn: conn, controlWriteMu: &sync.Mutex{}, results: make(chan resultEnvelope, 2), notifyResults: make(chan notifyResultEnvelope, 2), leases: map[string]string{"stale": "lease-stale"}}
|
||||||
|
r.results <- resultEnvelope{job: wire.CheckJob{JobID: "done", LeaseToken: "lease-done"}, reports: []wire.CheckResultReport{{JobID: "done"}}}
|
||||||
|
r.drainFinalResults()
|
||||||
|
found := false
|
||||||
|
deadline := time.After(time.Second)
|
||||||
|
for !found {
|
||||||
|
select {
|
||||||
|
case message := <-reports:
|
||||||
|
if message.StaleLease != nil {
|
||||||
|
assert.Equal(t, "stale", message.StaleLease.JobID)
|
||||||
|
found = true
|
||||||
|
}
|
||||||
|
case <-deadline:
|
||||||
|
t.Fatal("missing stale lease report")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestEnqueueTaskMessageRejectsMismatchedEnvelopeJobIDs(t *testing.T) {
|
||||||
|
r := &Runner{jobQueue: make(chan wire.CheckJob, 1), results: make(chan resultEnvelope, 1), stopCh: make(chan struct{})}
|
||||||
|
message := wire.WorkerMessage{Kind: "task", TaskEnvelope: &wire.TaskEnvelope{
|
||||||
|
Type: wire.TaskTypeCheck,
|
||||||
|
JobID: "outer-job",
|
||||||
|
Job: &wire.CheckJob{JobID: "inner-job", LeaseToken: "lease-1", Kind: "http"},
|
||||||
|
}}
|
||||||
|
|
||||||
|
require.True(t, r.enqueueTaskMessage(message))
|
||||||
|
assert.Empty(t, r.jobQueue)
|
||||||
|
assert.Empty(t, r.results, "a task without an unambiguous job ID cannot be reported safely")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestEnqueueTaskMessageRejectsMissingEnvelopeJobID(t *testing.T) {
|
||||||
|
r := &Runner{jobQueue: make(chan wire.CheckJob, 1), results: make(chan resultEnvelope, 1), stopCh: make(chan struct{})}
|
||||||
|
message := wire.WorkerMessage{Kind: "task", TaskEnvelope: &wire.TaskEnvelope{
|
||||||
|
Type: wire.TaskTypeCheck,
|
||||||
|
Job: &wire.CheckJob{JobID: "inner-job", LeaseToken: "lease-1", Kind: "http"},
|
||||||
|
}}
|
||||||
|
|
||||||
|
require.True(t, r.enqueueTaskMessage(message))
|
||||||
|
assert.Empty(t, r.jobQueue)
|
||||||
|
assert.Empty(t, r.results, "a missing outer ID cannot be reported safely")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestEnqueueTaskMessageReportsMalformedCheckEnvelopeWithMatchingJobID(t *testing.T) {
|
||||||
|
r := &Runner{jobQueue: make(chan wire.CheckJob, 1), results: make(chan resultEnvelope, 1), stopCh: make(chan struct{})}
|
||||||
|
message := wire.WorkerMessage{Kind: "task", TaskEnvelope: &wire.TaskEnvelope{
|
||||||
|
Type: wire.TaskTypeNotification,
|
||||||
|
JobID: "job-1",
|
||||||
|
Job: &wire.CheckJob{JobID: "job-1", LeaseToken: "lease-1", CheckID: 4, MonitorID: 5, Kind: "http"},
|
||||||
|
}}
|
||||||
|
|
||||||
|
require.True(t, r.enqueueTaskMessage(message))
|
||||||
|
assert.Empty(t, r.jobQueue)
|
||||||
|
env := <-r.results
|
||||||
|
require.Len(t, env.reports, 1)
|
||||||
|
assert.Equal(t, "job-1", env.reports[0].JobID)
|
||||||
|
assert.Equal(t, "FAIL", env.reports[0].State)
|
||||||
|
require.NotNil(t, env.reports[0].Error)
|
||||||
|
assert.Equal(t, malformedTaskEnvelopeError, *env.reports[0].Error)
|
||||||
|
assert.Empty(t, r.results)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestEnqueueTaskMessageRejectsAmbiguousEnvelope(t *testing.T) {
|
||||||
|
r := &Runner{
|
||||||
|
jobQueue: make(chan wire.CheckJob, 1),
|
||||||
|
results: make(chan resultEnvelope, 1),
|
||||||
|
notifyQueue: make(chan wire.NotificationTask, 1),
|
||||||
|
notifyResults: make(chan notifyResultEnvelope, 1),
|
||||||
|
stopCh: make(chan struct{}),
|
||||||
|
}
|
||||||
|
message := wire.WorkerMessage{Kind: "task", TaskEnvelope: &wire.TaskEnvelope{
|
||||||
|
Type: wire.TaskTypeCheck,
|
||||||
|
JobID: "job-1",
|
||||||
|
Job: &wire.CheckJob{JobID: "job-1", LeaseToken: "lease-1", Kind: "http"},
|
||||||
|
Notify: &wire.NotificationTask{JobID: "job-1", LeaseToken: "lease-1"},
|
||||||
|
}}
|
||||||
|
|
||||||
|
require.True(t, r.enqueueTaskMessage(message))
|
||||||
|
assert.Empty(t, r.jobQueue)
|
||||||
|
assert.Empty(t, r.results)
|
||||||
|
assert.Empty(t, r.notifyQueue)
|
||||||
|
assert.Empty(t, r.notifyResults)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestEnqueueTaskMessageReportsUnsupportedCheckKind(t *testing.T) {
|
||||||
|
r := &Runner{jobQueue: make(chan wire.CheckJob, 1), results: make(chan resultEnvelope, 1), stopCh: make(chan struct{})}
|
||||||
|
message := wire.WorkerMessage{Kind: "task", TaskEnvelope: &wire.TaskEnvelope{
|
||||||
|
Type: wire.TaskTypeCheck,
|
||||||
|
JobID: "job-1",
|
||||||
|
Job: &wire.CheckJob{JobID: "job-1", LeaseToken: "lease-1", CheckID: 4, MonitorID: 5, Kind: "rkn"},
|
||||||
|
}}
|
||||||
|
|
||||||
|
require.True(t, r.enqueueTaskMessage(message))
|
||||||
|
assert.Empty(t, r.jobQueue, "unsupported work must never reach the executor")
|
||||||
|
env := <-r.results
|
||||||
|
require.Len(t, env.reports, 1)
|
||||||
|
report := env.reports[0]
|
||||||
|
assert.Equal(t, "job-1", report.JobID)
|
||||||
|
assert.Equal(t, int64(4), report.CheckID)
|
||||||
|
assert.Equal(t, int64(5), report.MonitorID)
|
||||||
|
assert.Equal(t, "FAIL", report.State)
|
||||||
|
require.NotNil(t, report.Error)
|
||||||
|
assert.Equal(t, "unsupported_kind: rkn", *report.Error)
|
||||||
|
assert.Empty(t, r.results, "each rejected task must generate one terminal result")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestEnqueueTaskMessageReportsMalformedNotificationForInvalidType(t *testing.T) {
|
||||||
|
r := &Runner{notifyQueue: make(chan wire.NotificationTask, 1), notifyResults: make(chan notifyResultEnvelope, 1), stopCh: make(chan struct{})}
|
||||||
|
message := wire.WorkerMessage{Kind: "task", TaskEnvelope: &wire.TaskEnvelope{
|
||||||
|
Type: "unknown",
|
||||||
|
JobID: "job-1",
|
||||||
|
Notify: &wire.NotificationTask{JobID: "job-1", LeaseToken: "lease-1", MessageID: 9, Method: "email"},
|
||||||
|
}}
|
||||||
|
|
||||||
|
require.True(t, r.enqueueTaskMessage(message))
|
||||||
|
assert.Empty(t, r.notifyQueue)
|
||||||
|
env := <-r.notifyResults
|
||||||
|
assert.Equal(t, "job-1", env.report.JobID)
|
||||||
|
assert.Equal(t, "lease-1", env.report.LeaseToken)
|
||||||
|
assert.Equal(t, int64(9), env.report.MessageID)
|
||||||
|
assert.Equal(t, wire.NotificationResultPermanent, env.report.Status)
|
||||||
|
require.NotNil(t, env.report.Error)
|
||||||
|
assert.Equal(t, malformedTaskEnvelopeError, *env.report.Error)
|
||||||
|
assert.Empty(t, r.notifyResults)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestEnqueueTaskMessageRejectsEmptyLeaseWithoutSideEffects(t *testing.T) {
|
||||||
|
cases := []struct {
|
||||||
|
name string
|
||||||
|
message wire.WorkerMessage
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "envelope check",
|
||||||
|
message: wire.WorkerMessage{Kind: "task", TaskEnvelope: &wire.TaskEnvelope{
|
||||||
|
Type: wire.TaskTypeCheck, JobID: "job-1", Job: &wire.CheckJob{JobID: "job-1", Kind: "http"},
|
||||||
|
}},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "envelope notification",
|
||||||
|
message: wire.WorkerMessage{Kind: "task", TaskEnvelope: &wire.TaskEnvelope{
|
||||||
|
Type: wire.TaskTypeNotification, JobID: "job-1", Notify: &wire.NotificationTask{JobID: "job-1", Method: "email"},
|
||||||
|
}},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "legacy check",
|
||||||
|
message: wire.WorkerMessage{Kind: "task", Task: &wire.CheckJob{JobID: "job-1", Kind: "http"}},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "legacy notification",
|
||||||
|
message: wire.WorkerMessage{Kind: "task", NotificationTask: &wire.NotificationTask{JobID: "job-1", Method: "email"}},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
for _, tc := range cases {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
r := &Runner{
|
||||||
|
jobQueue: make(chan wire.CheckJob, 1),
|
||||||
|
results: make(chan resultEnvelope, 1),
|
||||||
|
notifyQueue: make(chan wire.NotificationTask, 1),
|
||||||
|
notifyResults: make(chan notifyResultEnvelope, 1),
|
||||||
|
stopCh: make(chan struct{}),
|
||||||
|
}
|
||||||
|
|
||||||
|
require.True(t, r.enqueueTaskMessage(tc.message))
|
||||||
|
assert.Empty(t, r.jobQueue)
|
||||||
|
assert.Empty(t, r.results)
|
||||||
|
assert.Empty(t, r.notifyQueue)
|
||||||
|
assert.Empty(t, r.notifyResults)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestEnqueueTaskMessageRejectsCrossAccountTask(t *testing.T) {
|
||||||
|
r := &Runner{jobQueue: make(chan wire.CheckJob, 1), results: make(chan resultEnvelope, 1), stopCh: make(chan struct{}), state: workerState{AccountID: 4}}
|
||||||
|
require.True(t, r.enqueueTaskMessage(wire.WorkerMessage{Kind: "task", Task: &wire.CheckJob{JobID: "job", LeaseToken: "lease", AccountID: 5}}))
|
||||||
|
env := <-r.results
|
||||||
|
require.NotNil(t, env.reports[0].Error)
|
||||||
|
assert.Equal(t, "account_scope_mismatch", *env.reports[0].Error)
|
||||||
|
}
|
||||||
|
|||||||
@@ -1,6 +1,9 @@
|
|||||||
package distworker
|
package distworker
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
"strconv"
|
"strconv"
|
||||||
"sync"
|
"sync"
|
||||||
"sync/atomic"
|
"sync/atomic"
|
||||||
@@ -8,6 +11,7 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/Jeffail/tunny"
|
"github.com/Jeffail/tunny"
|
||||||
|
"github.com/gorilla/websocket"
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
|
|
||||||
@@ -174,6 +178,835 @@ func TestEnqueueRespectsBackpressure(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestStopRejectsTerminalResultsWithoutClosingResultChannel(t *testing.T) {
|
||||||
|
r := NewRunner(&Config{MaxConcurrency: 1})
|
||||||
|
r.results = make(chan resultEnvelope, 1)
|
||||||
|
r.Stop()
|
||||||
|
|
||||||
|
assert.False(t, r.enqueueFailedCheck(wire.CheckJob{JobID: "job-1", LeaseToken: "lease-1"}, "unsupported_kind: rkn"))
|
||||||
|
assert.Empty(t, r.results)
|
||||||
|
|
||||||
|
select {
|
||||||
|
case r.results <- resultEnvelope{}:
|
||||||
|
default:
|
||||||
|
t.Fatal("results channel should remain open after Stop")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRotateTokenReconnectsWithoutStoppingRunner(t *testing.T) {
|
||||||
|
const (
|
||||||
|
oldToken = "old-token"
|
||||||
|
newToken = "new-token"
|
||||||
|
)
|
||||||
|
|
||||||
|
connections := make(chan string, 2)
|
||||||
|
results := make(chan wire.WorkerMessage, 1)
|
||||||
|
upgrader := websocket.Upgrader{CheckOrigin: func(*http.Request) bool { return true }}
|
||||||
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
|
||||||
|
switch req.URL.Path {
|
||||||
|
case "/api/internal/workers/rotate-token":
|
||||||
|
if req.Header.Get("Authorization") != "Bearer "+oldToken {
|
||||||
|
http.Error(w, "unexpected rotation token", http.StatusUnauthorized)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
_ = json.NewEncoder(w).Encode(struct {
|
||||||
|
AuthToken string `json:"auth_token"`
|
||||||
|
}{AuthToken: newToken})
|
||||||
|
case "/worker":
|
||||||
|
conn, err := upgrader.Upgrade(w, req, nil)
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
defer conn.Close()
|
||||||
|
token := req.URL.Query().Get("token")
|
||||||
|
connections <- token
|
||||||
|
if token == oldToken {
|
||||||
|
_, _, _ = conn.ReadMessage() // Rotation must close this connection.
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if token != newToken {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if conn.WriteJSON(wire.WorkerMessage{Kind: "task", Task: &wire.CheckJob{
|
||||||
|
JobID: "after-rotation", LeaseToken: "lease", CheckID: 1, Kind: "http",
|
||||||
|
}}) != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
for {
|
||||||
|
var message wire.WorkerMessage
|
||||||
|
if err := conn.ReadJSON(&message); err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if message.Kind == "result" && message.Result != nil && message.Result.JobID == "after-rotation" {
|
||||||
|
results <- message
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
http.NotFound(w, req)
|
||||||
|
}
|
||||||
|
}))
|
||||||
|
defer server.Close()
|
||||||
|
|
||||||
|
r := NewRunner(&Config{URL: server.URL, Token: oldToken, MaxConcurrency: 1})
|
||||||
|
r.executor = func(payload interface{}) interface{} {
|
||||||
|
job := payload.(wire.CheckJob)
|
||||||
|
return []wire.CheckResultReport{{JobID: job.JobID, CheckID: job.CheckID, State: "OK"}}
|
||||||
|
}
|
||||||
|
startDone := make(chan error, 1)
|
||||||
|
go func() { startDone <- r.Start() }()
|
||||||
|
t.Cleanup(func() {
|
||||||
|
r.Stop()
|
||||||
|
select {
|
||||||
|
case err := <-startDone:
|
||||||
|
require.NoError(t, err)
|
||||||
|
case <-time.After(time.Second):
|
||||||
|
t.Fatal("runner did not stop")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
select {
|
||||||
|
case token := <-connections:
|
||||||
|
require.Equal(t, oldToken, token)
|
||||||
|
case <-time.After(time.Second):
|
||||||
|
t.Fatal("worker did not establish its initial control connection")
|
||||||
|
}
|
||||||
|
|
||||||
|
gotToken, err := r.RotateToken(t.Context())
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.Equal(t, newToken, gotToken)
|
||||||
|
require.Equal(t, newToken, r.Token())
|
||||||
|
|
||||||
|
select {
|
||||||
|
case token := <-connections:
|
||||||
|
require.Equal(t, newToken, token)
|
||||||
|
case <-time.After(time.Second):
|
||||||
|
t.Fatal("worker did not reconnect with the replacement token")
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case result := <-results:
|
||||||
|
require.NotNil(t, result.Result)
|
||||||
|
assert.Equal(t, "OK", result.Result.State)
|
||||||
|
case <-time.After(time.Second):
|
||||||
|
t.Fatal("runner did not execute work after token rotation")
|
||||||
|
}
|
||||||
|
assert.False(t, r.stopped(), "rotation must not stop runner-owned subsystems")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestStopClosesAndJoinsIdleControlConnection(t *testing.T) {
|
||||||
|
closed := make(chan struct{}, 1)
|
||||||
|
connected := make(chan struct{}, 1)
|
||||||
|
upgrader := websocket.Upgrader{CheckOrigin: func(*http.Request) bool { return true }}
|
||||||
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
|
||||||
|
if req.URL.Path != "/worker" {
|
||||||
|
http.NotFound(w, req)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
conn, err := upgrader.Upgrade(w, req, nil)
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
defer conn.Close()
|
||||||
|
connected <- struct{}{}
|
||||||
|
_, _, _ = conn.ReadMessage()
|
||||||
|
closed <- struct{}{}
|
||||||
|
}))
|
||||||
|
defer server.Close()
|
||||||
|
|
||||||
|
r := NewRunner(&Config{URL: server.URL, Token: "token", MaxConcurrency: 1})
|
||||||
|
startDone := make(chan error, 1)
|
||||||
|
go func() { startDone <- r.Start() }()
|
||||||
|
t.Cleanup(func() { r.Stop() })
|
||||||
|
|
||||||
|
select {
|
||||||
|
case <-time.After(time.Second):
|
||||||
|
t.Fatal("worker did not establish idle control connection")
|
||||||
|
case <-connected:
|
||||||
|
}
|
||||||
|
r.Stop()
|
||||||
|
select {
|
||||||
|
case <-closed:
|
||||||
|
case <-time.After(time.Second):
|
||||||
|
t.Fatal("Stop did not close the idle control connection")
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case err := <-startDone:
|
||||||
|
require.NoError(t, err)
|
||||||
|
case <-time.After(time.Second):
|
||||||
|
t.Fatal("Stop did not join the control loop")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestStopCancelsDialInProgress(t *testing.T) {
|
||||||
|
dialStarted := make(chan struct{})
|
||||||
|
allowUpgrade := make(chan struct{})
|
||||||
|
upgrader := websocket.Upgrader{CheckOrigin: func(*http.Request) bool { return true }}
|
||||||
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
|
||||||
|
if req.URL.Path != "/worker" {
|
||||||
|
http.NotFound(w, req)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
close(dialStarted)
|
||||||
|
<-allowUpgrade
|
||||||
|
_, _ = upgrader.Upgrade(w, req, nil)
|
||||||
|
}))
|
||||||
|
defer func() {
|
||||||
|
close(allowUpgrade)
|
||||||
|
server.Close()
|
||||||
|
}()
|
||||||
|
|
||||||
|
r := NewRunner(&Config{URL: server.URL, Token: "token", MaxConcurrency: 1})
|
||||||
|
startDone := make(chan error, 1)
|
||||||
|
go func() { startDone <- r.Start() }()
|
||||||
|
select {
|
||||||
|
case <-dialStarted:
|
||||||
|
case <-time.After(time.Second):
|
||||||
|
t.Fatal("worker did not begin websocket dial")
|
||||||
|
}
|
||||||
|
r.Stop()
|
||||||
|
select {
|
||||||
|
case err := <-startDone:
|
||||||
|
require.NoError(t, err)
|
||||||
|
case <-time.After(time.Second):
|
||||||
|
t.Fatal("Stop did not join a canceled websocket dial")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestStartAndStopRegisterControlLoopSafely(t *testing.T) {
|
||||||
|
server := httptest.NewServer(http.NotFoundHandler())
|
||||||
|
defer server.Close()
|
||||||
|
for i := 0; i < 25; i++ {
|
||||||
|
r := NewRunner(&Config{URL: server.URL, Token: "token", MaxConcurrency: 1})
|
||||||
|
startDone := make(chan error, 1)
|
||||||
|
stopDone := make(chan struct{})
|
||||||
|
go func() { startDone <- r.Start() }()
|
||||||
|
go func() {
|
||||||
|
r.Stop()
|
||||||
|
close(stopDone)
|
||||||
|
}()
|
||||||
|
select {
|
||||||
|
case <-stopDone:
|
||||||
|
case <-time.After(time.Second):
|
||||||
|
t.Fatal("Stop did not complete")
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case <-startDone:
|
||||||
|
case <-time.After(time.Second):
|
||||||
|
t.Fatal("Start did not return after concurrent Stop")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRotateTokenPreservesDequeuedResult(t *testing.T) {
|
||||||
|
const (
|
||||||
|
oldToken = "old-token"
|
||||||
|
newToken = "new-token"
|
||||||
|
)
|
||||||
|
connected := make(chan string, 2)
|
||||||
|
delivered := make(chan wire.WorkerMessage, 1)
|
||||||
|
upgrader := websocket.Upgrader{CheckOrigin: func(*http.Request) bool { return true }}
|
||||||
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
|
||||||
|
switch req.URL.Path {
|
||||||
|
case "/api/internal/workers/rotate-token":
|
||||||
|
_ = json.NewEncoder(w).Encode(struct {
|
||||||
|
AuthToken string `json:"auth_token"`
|
||||||
|
}{AuthToken: newToken})
|
||||||
|
case "/worker":
|
||||||
|
conn, err := upgrader.Upgrade(w, req, nil)
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
defer conn.Close()
|
||||||
|
token := req.URL.Query().Get("token")
|
||||||
|
connected <- token
|
||||||
|
if token == oldToken {
|
||||||
|
var message wire.WorkerMessage
|
||||||
|
if conn.ReadJSON(&message) == nil {
|
||||||
|
delivered <- message
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if token != newToken {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
var message wire.WorkerMessage
|
||||||
|
if conn.ReadJSON(&message) == nil {
|
||||||
|
delivered <- message
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}))
|
||||||
|
defer server.Close()
|
||||||
|
|
||||||
|
r := NewRunner(&Config{URL: server.URL, Token: oldToken, MaxConcurrency: 1})
|
||||||
|
enteredWrite := make(chan struct{})
|
||||||
|
releaseWrite := make(chan struct{})
|
||||||
|
var once sync.Once
|
||||||
|
r.beforeControlWrite = func() {
|
||||||
|
once.Do(func() {
|
||||||
|
close(enteredWrite)
|
||||||
|
<-releaseWrite
|
||||||
|
})
|
||||||
|
}
|
||||||
|
startDone := make(chan error, 1)
|
||||||
|
go func() { startDone <- r.Start() }()
|
||||||
|
t.Cleanup(func() {
|
||||||
|
r.Stop()
|
||||||
|
select {
|
||||||
|
case <-startDone:
|
||||||
|
case <-time.After(time.Second):
|
||||||
|
t.Fatal("runner did not stop")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
select {
|
||||||
|
case token := <-connected:
|
||||||
|
require.Equal(t, oldToken, token)
|
||||||
|
case <-time.After(time.Second):
|
||||||
|
t.Fatal("worker did not establish its initial control connection")
|
||||||
|
}
|
||||||
|
r.results <- resultEnvelope{job: wire.CheckJob{JobID: "result", LeaseToken: "lease"}, reports: []wire.CheckResultReport{{JobID: "result", State: "OK"}}}
|
||||||
|
select {
|
||||||
|
case <-enteredWrite:
|
||||||
|
case <-time.After(time.Second):
|
||||||
|
t.Fatal("writer did not dequeue result")
|
||||||
|
}
|
||||||
|
rotated := make(chan error, 1)
|
||||||
|
go func() {
|
||||||
|
_, err := r.RotateToken(t.Context())
|
||||||
|
rotated <- err
|
||||||
|
}()
|
||||||
|
close(releaseWrite)
|
||||||
|
require.NoError(t, <-rotated)
|
||||||
|
|
||||||
|
select {
|
||||||
|
case message := <-delivered:
|
||||||
|
require.NotNil(t, message.Result)
|
||||||
|
assert.Equal(t, "result", message.Result.JobID)
|
||||||
|
assert.Equal(t, "lease", message.Result.LeaseToken)
|
||||||
|
case <-time.After(time.Second):
|
||||||
|
t.Fatal("dequeued result was lost during rotation")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRotateTokenSerializesWithStop(t *testing.T) {
|
||||||
|
rotationStarted := make(chan struct{})
|
||||||
|
releaseHandler := make(chan struct{})
|
||||||
|
upgrader := websocket.Upgrader{CheckOrigin: func(*http.Request) bool { return true }}
|
||||||
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
|
||||||
|
switch req.URL.Path {
|
||||||
|
case "/api/internal/workers/rotate-token":
|
||||||
|
close(rotationStarted)
|
||||||
|
<-releaseHandler
|
||||||
|
case "/worker":
|
||||||
|
conn, err := upgrader.Upgrade(w, req, nil)
|
||||||
|
if err == nil {
|
||||||
|
defer conn.Close()
|
||||||
|
_, _, _ = conn.ReadMessage()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}))
|
||||||
|
defer func() {
|
||||||
|
close(releaseHandler)
|
||||||
|
server.Close()
|
||||||
|
}()
|
||||||
|
|
||||||
|
r := NewRunner(&Config{URL: server.URL, Token: "old-token", MaxConcurrency: 1})
|
||||||
|
startDone := make(chan error, 1)
|
||||||
|
go func() { startDone <- r.Start() }()
|
||||||
|
t.Cleanup(func() { r.Stop() })
|
||||||
|
|
||||||
|
// Wait for Start to install its client before beginning rotation.
|
||||||
|
deadline := time.After(time.Second)
|
||||||
|
for {
|
||||||
|
r.clientMu.Lock()
|
||||||
|
started := r.client != nil
|
||||||
|
r.clientMu.Unlock()
|
||||||
|
if started {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case <-deadline:
|
||||||
|
t.Fatal("runner did not start")
|
||||||
|
default:
|
||||||
|
time.Sleep(time.Millisecond)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
rotated := make(chan error, 1)
|
||||||
|
go func() {
|
||||||
|
_, err := r.RotateToken(t.Context())
|
||||||
|
rotated <- err
|
||||||
|
}()
|
||||||
|
select {
|
||||||
|
case <-rotationStarted:
|
||||||
|
case <-time.After(time.Second):
|
||||||
|
t.Fatal("rotation request did not start")
|
||||||
|
}
|
||||||
|
stopped := make(chan struct{})
|
||||||
|
go func() {
|
||||||
|
r.Stop()
|
||||||
|
close(stopped)
|
||||||
|
}()
|
||||||
|
require.Error(t, <-rotated, "Stop must cancel an in-flight rotation request")
|
||||||
|
select {
|
||||||
|
case <-stopped:
|
||||||
|
case <-time.After(time.Second):
|
||||||
|
t.Fatal("Stop did not complete after canceling rotation")
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case err := <-startDone:
|
||||||
|
require.NoError(t, err)
|
||||||
|
case <-time.After(time.Second):
|
||||||
|
t.Fatal("runner did not stop")
|
||||||
|
}
|
||||||
|
_, err := r.RotateToken(t.Context())
|
||||||
|
require.Error(t, err, "rotation cannot succeed after shutdown")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestStopUnblocksRotationWaitingForWriter(t *testing.T) {
|
||||||
|
upgrader := websocket.Upgrader{CheckOrigin: func(*http.Request) bool { return true }}
|
||||||
|
connected := make(chan struct{}, 1)
|
||||||
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
|
||||||
|
switch req.URL.Path {
|
||||||
|
case "/api/internal/workers/rotate-token":
|
||||||
|
_ = json.NewEncoder(w).Encode(struct {
|
||||||
|
AuthToken string `json:"auth_token"`
|
||||||
|
}{AuthToken: "new-token"})
|
||||||
|
case "/worker":
|
||||||
|
conn, err := upgrader.Upgrade(w, req, nil)
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
defer conn.Close()
|
||||||
|
connected <- struct{}{}
|
||||||
|
_, _, _ = conn.ReadMessage()
|
||||||
|
}
|
||||||
|
}))
|
||||||
|
defer server.Close()
|
||||||
|
|
||||||
|
r := NewRunner(&Config{URL: server.URL, Token: "old-token", MaxConcurrency: 1})
|
||||||
|
writeBlocked := make(chan struct{})
|
||||||
|
var once sync.Once
|
||||||
|
r.beforeControlWrite = func() {
|
||||||
|
once.Do(func() {
|
||||||
|
close(writeBlocked)
|
||||||
|
<-r.controlCtx.Done()
|
||||||
|
})
|
||||||
|
}
|
||||||
|
rotationReady := make(chan struct{})
|
||||||
|
r.beforeTokenCommit = func() { close(rotationReady) }
|
||||||
|
startDone := make(chan error, 1)
|
||||||
|
go func() { startDone <- r.Start() }()
|
||||||
|
t.Cleanup(func() { r.Stop() })
|
||||||
|
select {
|
||||||
|
case <-connected:
|
||||||
|
case <-time.After(time.Second):
|
||||||
|
t.Fatal("worker did not connect")
|
||||||
|
}
|
||||||
|
r.results <- resultEnvelope{job: wire.CheckJob{JobID: "blocked", LeaseToken: "lease"}, reports: []wire.CheckResultReport{{JobID: "blocked", State: "OK"}}}
|
||||||
|
select {
|
||||||
|
case <-writeBlocked:
|
||||||
|
case <-time.After(time.Second):
|
||||||
|
t.Fatal("writer did not block")
|
||||||
|
}
|
||||||
|
rotated := make(chan error, 1)
|
||||||
|
go func() {
|
||||||
|
_, err := r.RotateToken(t.Context())
|
||||||
|
rotated <- err
|
||||||
|
}()
|
||||||
|
select {
|
||||||
|
case <-rotationReady:
|
||||||
|
case <-time.After(time.Second):
|
||||||
|
t.Fatal("rotation did not reach writer serialization")
|
||||||
|
}
|
||||||
|
stopped := make(chan struct{})
|
||||||
|
go func() {
|
||||||
|
r.Stop()
|
||||||
|
close(stopped)
|
||||||
|
}()
|
||||||
|
select {
|
||||||
|
case <-stopped:
|
||||||
|
case <-time.After(time.Second):
|
||||||
|
t.Fatal("Stop deadlocked behind rotation waiting for writer")
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case <-rotated:
|
||||||
|
case <-time.After(time.Second):
|
||||||
|
t.Fatal("rotation did not unblock after Stop closed the connection")
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case err := <-startDone:
|
||||||
|
require.NoError(t, err)
|
||||||
|
case <-time.After(time.Second):
|
||||||
|
t.Fatal("runner did not stop")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestStopWinsBeforePostHTTPRotationCommit(t *testing.T) {
|
||||||
|
commitReady := make(chan struct{})
|
||||||
|
releaseCommit := make(chan struct{})
|
||||||
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
|
||||||
|
if req.URL.Path != "/api/internal/workers/rotate-token" {
|
||||||
|
http.NotFound(w, req)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
_ = json.NewEncoder(w).Encode(struct {
|
||||||
|
AuthToken string `json:"auth_token"`
|
||||||
|
}{AuthToken: "new-token"})
|
||||||
|
}))
|
||||||
|
defer server.Close()
|
||||||
|
|
||||||
|
r := NewRunner(&Config{URL: server.URL, Token: "old-token", MaxConcurrency: 1})
|
||||||
|
r.client = NewClient(server.URL, "old-token")
|
||||||
|
r.beforeTokenCommit = func() {
|
||||||
|
close(commitReady)
|
||||||
|
<-releaseCommit
|
||||||
|
}
|
||||||
|
rotated := make(chan error, 1)
|
||||||
|
go func() {
|
||||||
|
_, err := r.RotateToken(t.Context())
|
||||||
|
rotated <- err
|
||||||
|
}()
|
||||||
|
select {
|
||||||
|
case <-commitReady:
|
||||||
|
case <-time.After(time.Second):
|
||||||
|
t.Fatal("rotation did not reach post-HTTP commit")
|
||||||
|
}
|
||||||
|
stopped := make(chan struct{})
|
||||||
|
go func() {
|
||||||
|
r.Stop()
|
||||||
|
close(stopped)
|
||||||
|
}()
|
||||||
|
select {
|
||||||
|
case <-r.stopCh:
|
||||||
|
case <-time.After(time.Second):
|
||||||
|
t.Fatal("Stop did not win lifecycle ownership")
|
||||||
|
}
|
||||||
|
close(releaseCommit)
|
||||||
|
require.Error(t, <-rotated, "rotation cannot succeed after Stop wins")
|
||||||
|
select {
|
||||||
|
case <-stopped:
|
||||||
|
case <-time.After(time.Second):
|
||||||
|
t.Fatal("Stop did not finish")
|
||||||
|
}
|
||||||
|
assert.Equal(t, "old-token", r.Token())
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRotateTokenUnchangedResponseReleasesLifecycle(t *testing.T) {
|
||||||
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
|
||||||
|
_ = json.NewEncoder(w).Encode(struct {
|
||||||
|
AuthToken string `json:"auth_token"`
|
||||||
|
}{AuthToken: "old-token"})
|
||||||
|
}))
|
||||||
|
defer server.Close()
|
||||||
|
|
||||||
|
r := NewRunner(&Config{URL: server.URL, Token: "old-token", MaxConcurrency: 1})
|
||||||
|
r.client = NewClient(server.URL, "old-token")
|
||||||
|
_, err := r.RotateToken(t.Context())
|
||||||
|
require.Error(t, err)
|
||||||
|
|
||||||
|
stopped := make(chan struct{})
|
||||||
|
go func() {
|
||||||
|
r.Stop()
|
||||||
|
close(stopped)
|
||||||
|
}()
|
||||||
|
select {
|
||||||
|
case <-stopped:
|
||||||
|
case <-time.After(time.Second):
|
||||||
|
t.Fatal("Stop deadlocked after unchanged rotation response")
|
||||||
|
}
|
||||||
|
_, err = r.RotateToken(t.Context())
|
||||||
|
require.Error(t, err, "later rotation must observe shutdown")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRotateTokenClosesStalledWriterBeforeWaiting(t *testing.T) {
|
||||||
|
upgrader := websocket.Upgrader{CheckOrigin: func(*http.Request) bool { return true }}
|
||||||
|
connected := make(chan struct{}, 1)
|
||||||
|
connectionClosed := make(chan struct{})
|
||||||
|
resent := make(chan wire.WorkerMessage, 1)
|
||||||
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
|
||||||
|
switch req.URL.Path {
|
||||||
|
case "/api/internal/workers/rotate-token":
|
||||||
|
_ = json.NewEncoder(w).Encode(struct {
|
||||||
|
AuthToken string `json:"auth_token"`
|
||||||
|
}{AuthToken: "new-token"})
|
||||||
|
case "/worker":
|
||||||
|
conn, err := upgrader.Upgrade(w, req, nil)
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
defer conn.Close()
|
||||||
|
if req.URL.Query().Get("token") == "old-token" {
|
||||||
|
connected <- struct{}{}
|
||||||
|
_, _, _ = conn.ReadMessage()
|
||||||
|
connectionClosed <- struct{}{}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
var message wire.WorkerMessage
|
||||||
|
if conn.ReadJSON(&message) == nil {
|
||||||
|
resent <- message
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}))
|
||||||
|
defer server.Close()
|
||||||
|
|
||||||
|
r := NewRunner(&Config{URL: server.URL, Token: "old-token", MaxConcurrency: 1})
|
||||||
|
writerBlocked := make(chan struct{})
|
||||||
|
var once sync.Once
|
||||||
|
r.beforeControlWrite = func() {
|
||||||
|
once.Do(func() {
|
||||||
|
close(writerBlocked)
|
||||||
|
<-connectionClosed
|
||||||
|
})
|
||||||
|
}
|
||||||
|
startDone := make(chan error, 1)
|
||||||
|
go func() { startDone <- r.Start() }()
|
||||||
|
t.Cleanup(func() { r.Stop() })
|
||||||
|
select {
|
||||||
|
case <-connected:
|
||||||
|
case <-time.After(time.Second):
|
||||||
|
t.Fatal("worker did not connect")
|
||||||
|
}
|
||||||
|
r.results <- resultEnvelope{job: wire.CheckJob{JobID: "stalled", LeaseToken: "lease"}, reports: []wire.CheckResultReport{{JobID: "stalled", State: "OK"}}}
|
||||||
|
select {
|
||||||
|
case <-writerBlocked:
|
||||||
|
case <-time.After(time.Second):
|
||||||
|
t.Fatal("writer did not stall")
|
||||||
|
}
|
||||||
|
rotated := make(chan error, 1)
|
||||||
|
go func() {
|
||||||
|
_, err := r.RotateToken(t.Context())
|
||||||
|
rotated <- err
|
||||||
|
}()
|
||||||
|
select {
|
||||||
|
case err := <-rotated:
|
||||||
|
require.NoError(t, err)
|
||||||
|
case <-time.After(time.Second):
|
||||||
|
t.Fatal("rotation waited for stalled writer before closing its connection")
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case message := <-resent:
|
||||||
|
require.NotNil(t, message.Result)
|
||||||
|
assert.Equal(t, "stalled", message.Result.JobID)
|
||||||
|
case <-time.After(time.Second):
|
||||||
|
t.Fatal("failed check result was not requeued after rotation")
|
||||||
|
}
|
||||||
|
r.Stop()
|
||||||
|
select {
|
||||||
|
case err := <-startDone:
|
||||||
|
require.NoError(t, err)
|
||||||
|
case <-time.After(time.Second):
|
||||||
|
t.Fatal("runner did not stop")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestWriterDropsFailedServerMetricSnapshot(t *testing.T) {
|
||||||
|
upgrader := websocket.Upgrader{CheckOrigin: func(*http.Request) bool { return true }}
|
||||||
|
upgraded := make(chan struct{})
|
||||||
|
closeServer := make(chan struct{})
|
||||||
|
serverClosed := make(chan struct{})
|
||||||
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
|
||||||
|
conn, err := upgrader.Upgrade(w, req, nil)
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
defer conn.Close()
|
||||||
|
close(upgraded)
|
||||||
|
<-closeServer
|
||||||
|
close(serverClosed)
|
||||||
|
}))
|
||||||
|
defer server.Close()
|
||||||
|
|
||||||
|
conn, err := NewClient(server.URL, "token").WorkerSocket()
|
||||||
|
require.NoError(t, err)
|
||||||
|
defer conn.Close()
|
||||||
|
select {
|
||||||
|
case <-upgraded:
|
||||||
|
case <-time.After(time.Second):
|
||||||
|
t.Fatal("websocket did not connect")
|
||||||
|
}
|
||||||
|
|
||||||
|
r := NewRunner(&Config{})
|
||||||
|
r.metricResults = make(chan metricEnvelope, 1)
|
||||||
|
enteredWrite := make(chan struct{})
|
||||||
|
releaseWrite := make(chan struct{})
|
||||||
|
r.beforeControlWrite = func() {
|
||||||
|
close(enteredWrite)
|
||||||
|
<-releaseWrite
|
||||||
|
}
|
||||||
|
done := make(chan struct{})
|
||||||
|
writerDone := make(chan struct{})
|
||||||
|
go func() {
|
||||||
|
var writeMu sync.Mutex
|
||||||
|
r.writer(conn, &writeMu, done, 1)
|
||||||
|
close(writerDone)
|
||||||
|
}()
|
||||||
|
r.metricResults <- metricEnvelope{generation: 1, report: wire.ServerMetricReport{ServerID: 1}}
|
||||||
|
select {
|
||||||
|
case <-enteredWrite:
|
||||||
|
case <-time.After(time.Second):
|
||||||
|
t.Fatal("writer did not dequeue metric snapshot")
|
||||||
|
}
|
||||||
|
close(closeServer)
|
||||||
|
select {
|
||||||
|
case <-serverClosed:
|
||||||
|
case <-time.After(time.Second):
|
||||||
|
t.Fatal("server did not close websocket")
|
||||||
|
}
|
||||||
|
_ = conn.Close()
|
||||||
|
close(releaseWrite)
|
||||||
|
select {
|
||||||
|
case <-writerDone:
|
||||||
|
case <-time.After(time.Second):
|
||||||
|
t.Fatal("writer did not return after failed metric write")
|
||||||
|
}
|
||||||
|
_, replayable := r.takeOutbox()
|
||||||
|
assert.False(t, replayable, "failed metric snapshot must not enter result outbox")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestWriterRequeuesFailedNotificationResult(t *testing.T) {
|
||||||
|
upgrader := websocket.Upgrader{CheckOrigin: func(*http.Request) bool { return true }}
|
||||||
|
upgraded := make(chan struct{})
|
||||||
|
closeServer := make(chan struct{})
|
||||||
|
serverClosed := make(chan struct{})
|
||||||
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
|
||||||
|
conn, err := upgrader.Upgrade(w, req, nil)
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
defer conn.Close()
|
||||||
|
close(upgraded)
|
||||||
|
<-closeServer
|
||||||
|
close(serverClosed)
|
||||||
|
}))
|
||||||
|
defer server.Close()
|
||||||
|
|
||||||
|
conn, err := NewClient(server.URL, "token").WorkerSocket()
|
||||||
|
require.NoError(t, err)
|
||||||
|
defer conn.Close()
|
||||||
|
select {
|
||||||
|
case <-upgraded:
|
||||||
|
case <-time.After(time.Second):
|
||||||
|
t.Fatal("websocket did not connect")
|
||||||
|
}
|
||||||
|
|
||||||
|
r := NewRunner(&Config{})
|
||||||
|
r.notifyResults = make(chan notifyResultEnvelope, 1)
|
||||||
|
enteredWrite := make(chan struct{})
|
||||||
|
releaseWrite := make(chan struct{})
|
||||||
|
r.beforeControlWrite = func() {
|
||||||
|
close(enteredWrite)
|
||||||
|
<-releaseWrite
|
||||||
|
}
|
||||||
|
done := make(chan struct{})
|
||||||
|
writerDone := make(chan struct{})
|
||||||
|
go func() {
|
||||||
|
var writeMu sync.Mutex
|
||||||
|
r.writer(conn, &writeMu, done, 0)
|
||||||
|
close(writerDone)
|
||||||
|
}()
|
||||||
|
r.notifyResults <- notifyResultEnvelope{report: wire.NotificationResultReport{JobID: "notification", LeaseToken: "lease"}}
|
||||||
|
select {
|
||||||
|
case <-enteredWrite:
|
||||||
|
case <-time.After(time.Second):
|
||||||
|
t.Fatal("writer did not dequeue notification result")
|
||||||
|
}
|
||||||
|
close(closeServer)
|
||||||
|
select {
|
||||||
|
case <-serverClosed:
|
||||||
|
case <-time.After(time.Second):
|
||||||
|
t.Fatal("server did not close websocket")
|
||||||
|
}
|
||||||
|
_ = conn.Close()
|
||||||
|
close(releaseWrite)
|
||||||
|
select {
|
||||||
|
case <-writerDone:
|
||||||
|
case <-time.After(time.Second):
|
||||||
|
t.Fatal("writer did not return after failed notification write")
|
||||||
|
}
|
||||||
|
message, replayable := r.takeOutbox()
|
||||||
|
require.True(t, replayable, "failed notification result must enter result outbox")
|
||||||
|
require.NotNil(t, message.NotificationResult)
|
||||||
|
assert.Equal(t, "notification", message.NotificationResult.JobID)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMetricGenerationRejectsDisconnectedAndSendsFreshMetric(t *testing.T) {
|
||||||
|
upgrader := websocket.Upgrader{CheckOrigin: func(*http.Request) bool { return true }}
|
||||||
|
firstClosed := make(chan struct{})
|
||||||
|
secondConnected := make(chan struct{})
|
||||||
|
received := make(chan wire.WorkerMessage, 1)
|
||||||
|
var connections atomic.Int32
|
||||||
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
|
||||||
|
conn, err := upgrader.Upgrade(w, req, nil)
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
defer conn.Close()
|
||||||
|
if connections.Add(1) == 1 {
|
||||||
|
close(firstClosed)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
close(secondConnected)
|
||||||
|
var message wire.WorkerMessage
|
||||||
|
if conn.ReadJSON(&message) == nil {
|
||||||
|
received <- message
|
||||||
|
}
|
||||||
|
}))
|
||||||
|
defer server.Close()
|
||||||
|
|
||||||
|
r := NewRunner(&Config{URL: server.URL, Token: "token", MaxConcurrency: 1})
|
||||||
|
startDone := make(chan error, 1)
|
||||||
|
go func() { startDone <- r.Start() }()
|
||||||
|
t.Cleanup(func() { r.Stop() })
|
||||||
|
select {
|
||||||
|
case <-firstClosed:
|
||||||
|
case <-time.After(time.Second):
|
||||||
|
t.Fatal("worker did not establish initial websocket")
|
||||||
|
}
|
||||||
|
// Wait until the old connection has fully torn down, then try to enqueue a
|
||||||
|
// metric in the old-drain/disconnected interleaving.
|
||||||
|
deadline := time.After(time.Second)
|
||||||
|
for r.metricGeneration.Load() != 0 {
|
||||||
|
select {
|
||||||
|
case <-deadline:
|
||||||
|
t.Fatal("old control generation did not clear")
|
||||||
|
default:
|
||||||
|
time.Sleep(time.Millisecond)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
staleCount := 1
|
||||||
|
r.enqueueMetric(wire.ServerMetricReport{ServerID: 1, ProcessCount: &staleCount})
|
||||||
|
assert.Empty(t, r.metricResults, "disconnected metric must not enter bounded channel")
|
||||||
|
select {
|
||||||
|
case r.reconnectCh <- struct{}{}:
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case <-secondConnected:
|
||||||
|
case <-time.After(time.Second):
|
||||||
|
t.Fatal("worker did not reconnect")
|
||||||
|
}
|
||||||
|
deadline = time.After(time.Second)
|
||||||
|
for r.metricGeneration.Load() == 0 {
|
||||||
|
select {
|
||||||
|
case <-deadline:
|
||||||
|
t.Fatal("new control generation did not install")
|
||||||
|
default:
|
||||||
|
time.Sleep(time.Millisecond)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// This enqueue occurs immediately after the new connection installation.
|
||||||
|
freshCount := 2
|
||||||
|
r.enqueueMetric(wire.ServerMetricReport{ServerID: 1, ProcessCount: &freshCount})
|
||||||
|
select {
|
||||||
|
case message := <-received:
|
||||||
|
require.NotNil(t, message.ServerMetric)
|
||||||
|
assert.Equal(t, 2, *message.ServerMetric.ProcessCount)
|
||||||
|
case <-time.After(time.Second):
|
||||||
|
t.Fatal("fresh metric was not sent after reconnect")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestApplyInitResizesPool(t *testing.T) {
|
func TestApplyInitResizesPool(t *testing.T) {
|
||||||
executor := func(payload interface{}) interface{} {
|
executor := func(payload interface{}) interface{} {
|
||||||
return []wire.CheckResultReport{}
|
return []wire.CheckResultReport{}
|
||||||
@@ -316,3 +1149,59 @@ func TestApplyInitStoresURLInMemory(t *testing.T) {
|
|||||||
assert.Equal(t, "", r.URL(),
|
assert.Equal(t, "", r.URL(),
|
||||||
"URL() must return empty after applyInit with empty URL")
|
"URL() must return empty after applyInit with empty URL")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestApplyInitPrefersPublicURLOverLegacyURL verifies the bounded-migration
|
||||||
|
// precedence on the init/config frame: PublicURL (canonical) wins whenever
|
||||||
|
// it is non-empty, and the legacy URL field remains the fallback for old
|
||||||
|
// control planes.
|
||||||
|
func TestApplyInitPrefersPublicURLOverLegacyURL(t *testing.T) {
|
||||||
|
executor := func(payload interface{}) interface{} {
|
||||||
|
return []wire.CheckResultReport{}
|
||||||
|
}
|
||||||
|
r := newTestRunner(t, 4, 1, executor)
|
||||||
|
|
||||||
|
r.applyInit(&wire.WorkerInit{
|
||||||
|
WorkerID: "w-1",
|
||||||
|
Concurrency: 2,
|
||||||
|
PublicURL: "https://canonical.example.com",
|
||||||
|
URL: "https://legacy.example.com",
|
||||||
|
})
|
||||||
|
assert.Equal(t, "https://canonical.example.com", r.URL(),
|
||||||
|
"PublicURL must win over the legacy URL field")
|
||||||
|
|
||||||
|
r.applyInit(&wire.WorkerInit{
|
||||||
|
WorkerID: "w-1",
|
||||||
|
Concurrency: 2,
|
||||||
|
URL: "https://legacy.example.com",
|
||||||
|
})
|
||||||
|
assert.Equal(t, "https://legacy.example.com", r.URL(),
|
||||||
|
"legacy URL field must be used when PublicURL is empty")
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestApplyInitInvalidAcceptedURLKeepsPrior verifies the safe-fallback
|
||||||
|
// behavior when the control plane supplies an unusable advertised URL:
|
||||||
|
// the previous accepted value is kept (never regressed to a garbage
|
||||||
|
// endpoint), while a valid empty init still clears it.
|
||||||
|
func TestApplyInitInvalidAcceptedURLKeepsPrior(t *testing.T) {
|
||||||
|
executor := func(payload interface{}) interface{} {
|
||||||
|
return []wire.CheckResultReport{}
|
||||||
|
}
|
||||||
|
r := newTestRunner(t, 4, 1, executor)
|
||||||
|
|
||||||
|
r.applyInit(&wire.WorkerInit{WorkerID: "w-1", Concurrency: 2, PublicURL: "https://worker.example.com"})
|
||||||
|
assert.Equal(t, "https://worker.example.com", r.URL())
|
||||||
|
|
||||||
|
// Invalid value: keep the previous accepted URL.
|
||||||
|
r.applyInit(&wire.WorkerInit{WorkerID: "w-1", Concurrency: 2, PublicURL: "https://:27401"})
|
||||||
|
assert.Equal(t, "https://worker.example.com", r.URL(),
|
||||||
|
"invalid accepted URL must not replace the stored value")
|
||||||
|
|
||||||
|
r.applyInit(&wire.WorkerInit{WorkerID: "w-1", Concurrency: 2, URL: "not a url"})
|
||||||
|
assert.Equal(t, "https://worker.example.com", r.URL(),
|
||||||
|
"invalid legacy url field must not replace the stored value")
|
||||||
|
|
||||||
|
// Valid empty init clears, as before.
|
||||||
|
r.applyInit(&wire.WorkerInit{WorkerID: "w-1", Concurrency: 2})
|
||||||
|
assert.Equal(t, "", r.URL(),
|
||||||
|
"valid empty init must clear the stored URL")
|
||||||
|
}
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
package distworker
|
package distworker
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"context"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"sort"
|
"sort"
|
||||||
@@ -35,31 +36,40 @@ const (
|
|||||||
// the real Linux /proc and statfs collector; unsupported platforms return no
|
// the real Linux /proc and statfs collector; unsupported platforms return no
|
||||||
// report rather than fabricated values. The worker has no control-plane DB
|
// report rather than fabricated values. The worker has no control-plane DB
|
||||||
// access and forwards snapshots on its authenticated websocket.
|
// access and forwards snapshots on its authenticated websocket.
|
||||||
func (r *Runner) serverMetricLoop() {
|
func (r *Runner) serverMetricLoop(ctx context.Context) {
|
||||||
ticker := time.NewTicker(serverMetricInterval)
|
ticker := time.NewTicker(serverMetricInterval)
|
||||||
defer ticker.Stop()
|
defer ticker.Stop()
|
||||||
for {
|
for {
|
||||||
select {
|
select {
|
||||||
case <-r.stopCh:
|
case <-ctx.Done():
|
||||||
return
|
return
|
||||||
case <-ticker.C:
|
case <-ticker.C:
|
||||||
serverID := r.serverID.Load()
|
serverID := r.serverID.Load()
|
||||||
if serverID == 0 || r.metricResults == nil {
|
if serverID == 0 {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
report, ok := collectServerMetric(serverID)
|
report, ok := collectServerMetric(serverID)
|
||||||
if !ok {
|
if !ok {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
select {
|
r.enqueueMetric(report)
|
||||||
case r.metricResults <- report:
|
}
|
||||||
case <-r.stopCh:
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *Runner) enqueueMetric(report wire.ServerMetricReport) {
|
||||||
|
if r.metricResults == nil {
|
||||||
return
|
return
|
||||||
|
}
|
||||||
|
generation := r.metricGeneration.Load()
|
||||||
|
if generation == 0 || r.metricGeneration.Load() != generation {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case r.metricResults <- metricEnvelope{generation: generation, report: report}:
|
||||||
default:
|
default:
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func collectServerMetric(serverID int64) (wire.ServerMetricReport, bool) {
|
func collectServerMetric(serverID int64) (wire.ServerMetricReport, bool) {
|
||||||
memTotal, memAvailable, load1, load5, load15, uptime, ok := readLinuxHostMetrics("/proc")
|
memTotal, memAvailable, load1, load5, load15, uptime, ok := readLinuxHostMetrics("/proc")
|
||||||
|
|||||||
98
internal/distworker/state.go
Обычный файл
98
internal/distworker/state.go
Обычный файл
@@ -0,0 +1,98 @@
|
|||||||
|
package distworker
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
)
|
||||||
|
|
||||||
|
type workerState struct {
|
||||||
|
Token string `json:"token"`
|
||||||
|
WorkerID string `json:"worker_id"`
|
||||||
|
VerificationKey string `json:"verification_key"`
|
||||||
|
SigningKeyID string `json:"signing_key_id"`
|
||||||
|
AccountID int64 `json:"account_id"`
|
||||||
|
ConfigVersion int64 `json:"config_version"`
|
||||||
|
ConfigPayload string `json:"config_payload"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func loadWorkerState(path string) (workerState, error) {
|
||||||
|
if path == "" {
|
||||||
|
return workerState{}, nil
|
||||||
|
}
|
||||||
|
if !filepath.IsAbs(path) {
|
||||||
|
return workerState{}, fmt.Errorf("worker state file must be absolute")
|
||||||
|
}
|
||||||
|
if info, err := os.Stat(filepath.Dir(path)); err != nil || !info.IsDir() || info.Mode().Perm() != 0o700 {
|
||||||
|
return workerState{}, fmt.Errorf("worker state directory must have mode 0700")
|
||||||
|
}
|
||||||
|
info, err := os.Stat(path)
|
||||||
|
if os.IsNotExist(err) {
|
||||||
|
return workerState{}, nil
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return workerState{}, err
|
||||||
|
}
|
||||||
|
if info.Mode().Perm() != 0o600 {
|
||||||
|
return workerState{}, fmt.Errorf("worker state file must have mode 0600")
|
||||||
|
}
|
||||||
|
data, err := os.ReadFile(path)
|
||||||
|
if err != nil {
|
||||||
|
return workerState{}, err
|
||||||
|
}
|
||||||
|
var state workerState
|
||||||
|
if err := json.Unmarshal(data, &state); err != nil {
|
||||||
|
return workerState{}, err
|
||||||
|
}
|
||||||
|
return state, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func saveWorkerState(path string, state workerState) error {
|
||||||
|
if path == "" {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if !filepath.IsAbs(path) {
|
||||||
|
return fmt.Errorf("worker state file must be absolute")
|
||||||
|
}
|
||||||
|
if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if info, err := os.Stat(filepath.Dir(path)); err != nil || !info.IsDir() || info.Mode().Perm() != 0o700 {
|
||||||
|
return fmt.Errorf("worker state directory must have mode 0700")
|
||||||
|
}
|
||||||
|
data, err := json.Marshal(state)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
tmp, err := os.CreateTemp(filepath.Dir(path), ".state-")
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
tmpName := tmp.Name()
|
||||||
|
defer os.Remove(tmpName)
|
||||||
|
if err := tmp.Chmod(0o600); err != nil {
|
||||||
|
_ = tmp.Close()
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if _, err := tmp.Write(data); err != nil {
|
||||||
|
_ = tmp.Close()
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := tmp.Sync(); err != nil {
|
||||||
|
_ = tmp.Close()
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := tmp.Close(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := os.Rename(tmpName, path); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
dir, err := os.Open(filepath.Dir(path))
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer dir.Close()
|
||||||
|
return dir.Sync()
|
||||||
|
}
|
||||||
33
internal/distworker/state_test.go
Обычный файл
33
internal/distworker/state_test.go
Обычный файл
@@ -0,0 +1,33 @@
|
|||||||
|
package distworker
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestWorkerStateRoundTripUsesPrivatePermissions(t *testing.T) {
|
||||||
|
dir := filepath.Join(t.TempDir(), "state")
|
||||||
|
require.NoError(t, os.Mkdir(dir, 0o700))
|
||||||
|
path := filepath.Join(dir, "state.json")
|
||||||
|
want := workerState{Token: "secret", WorkerID: "worker-1", VerificationKey: "key", SigningKeyID: "key-2026", AccountID: 7, ConfigVersion: 2}
|
||||||
|
require.NoError(t, saveWorkerState(path, want))
|
||||||
|
info, err := os.Stat(path)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, os.FileMode(0o600), info.Mode().Perm())
|
||||||
|
got, err := loadWorkerState(path)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, want, got)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestWorkerStateRejectsInsecurePermissions(t *testing.T) {
|
||||||
|
dir := filepath.Join(t.TempDir(), "state")
|
||||||
|
require.NoError(t, os.Mkdir(dir, 0o700))
|
||||||
|
path := filepath.Join(dir, "state.json")
|
||||||
|
require.NoError(t, os.WriteFile(path, []byte(`{"token":"secret"}`), 0o644))
|
||||||
|
_, err := loadWorkerState(path)
|
||||||
|
require.Error(t, err)
|
||||||
|
}
|
||||||
63
internal/distworker/trust_test.go
Обычный файл
63
internal/distworker/trust_test.go
Обычный файл
@@ -0,0 +1,63 @@
|
|||||||
|
package distworker
|
||||||
|
|
||||||
|
import (
|
||||||
|
"crypto/ed25519"
|
||||||
|
"crypto/rand"
|
||||||
|
"encoding/base64"
|
||||||
|
"encoding/json"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
"rocketgit.ru/rsmon/worker/internal/wire"
|
||||||
|
)
|
||||||
|
|
||||||
|
func signedInit(t *testing.T, private ed25519.PrivateKey, account, version int64) *wire.WorkerInit {
|
||||||
|
t.Helper()
|
||||||
|
init := &wire.WorkerInit{WorkerID: "w", SigningKeyID: "v1", AccountID: &account, ConfigVersion: version, ExpiresAt: time.Now().Add(time.Minute).Format(time.RFC3339Nano)}
|
||||||
|
body, err := json.Marshal(init)
|
||||||
|
require.NoError(t, err)
|
||||||
|
init.Signature = base64.StdEncoding.EncodeToString(ed25519.Sign(private, body))
|
||||||
|
return init
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestVerifyInitRejectsTamperWrongKeyAndAccountChange(t *testing.T) {
|
||||||
|
pub, priv, err := ed25519.GenerateKey(rand.Reader)
|
||||||
|
require.NoError(t, err)
|
||||||
|
r := NewRunner(&Config{})
|
||||||
|
r.state = workerState{WorkerID: "w", VerificationKey: base64.StdEncoding.EncodeToString(pub), SigningKeyID: "v1", AccountID: 1}
|
||||||
|
init := signedInit(t, priv, 1, 1)
|
||||||
|
require.NoError(t, r.verifyInit(init))
|
||||||
|
init.ConfigVersion = 2
|
||||||
|
assert.Error(t, r.verifyInit(init), "tamper must invalidate signature")
|
||||||
|
_, other, err := ed25519.GenerateKey(rand.Reader)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Error(t, r.verifyInit(signedInit(t, other, 1, 2)), "wrong key")
|
||||||
|
assert.Error(t, r.verifyInit(signedInit(t, priv, 2, 2)), "account scope must be immutable")
|
||||||
|
unknown := signedInit(t, priv, 1, 2)
|
||||||
|
unknown.SigningKeyID = "unknown"
|
||||||
|
assert.Error(t, r.verifyInit(unknown), "unknown key id")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestVerifyInitAcceptsIdenticalRestartReplayOnly(t *testing.T) {
|
||||||
|
pub, priv, err := ed25519.GenerateKey(rand.Reader)
|
||||||
|
require.NoError(t, err)
|
||||||
|
first := NewRunner(&Config{})
|
||||||
|
first.state = workerState{WorkerID: "w", VerificationKey: base64.StdEncoding.EncodeToString(pub), SigningKeyID: "v1", AccountID: 1}
|
||||||
|
init := signedInit(t, priv, 1, 4)
|
||||||
|
require.NoError(t, first.verifyInit(init))
|
||||||
|
// Restart restores only durable identity state. The exact verified snapshot
|
||||||
|
// remains safe to replay at the same generation.
|
||||||
|
restarted := NewRunner(&Config{})
|
||||||
|
restarted.state = first.state
|
||||||
|
require.NoError(t, restarted.verifyInit(signedInit(t, priv, 1, 4)))
|
||||||
|
assert.Error(t, restarted.verifyInit(signedInit(t, priv, 1, 3)), "older generation")
|
||||||
|
different := signedInit(t, priv, 1, 4)
|
||||||
|
different.RegionCode = "other"
|
||||||
|
body, err := json.Marshal(&wire.WorkerInit{WorkerID: different.WorkerID, SigningKeyID: different.SigningKeyID, AccountID: different.AccountID, ConfigVersion: different.ConfigVersion, ExpiresAt: different.ExpiresAt, RegionCode: different.RegionCode})
|
||||||
|
require.NoError(t, err)
|
||||||
|
different.Signature = base64.StdEncoding.EncodeToString(ed25519.Sign(priv, body))
|
||||||
|
assert.Error(t, restarted.verifyInit(different), "same generation different content")
|
||||||
|
require.NoError(t, restarted.verifyInit(signedInit(t, priv, 1, 5)), "newer generation")
|
||||||
|
}
|
||||||
357
internal/installer/deploy.go
Обычный файл
357
internal/installer/deploy.go
Обычный файл
@@ -0,0 +1,357 @@
|
|||||||
|
package installer
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"encoding/base64"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"net"
|
||||||
|
"os"
|
||||||
|
"os/user"
|
||||||
|
"path/filepath"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"golang.org/x/crypto/ssh"
|
||||||
|
"golang.org/x/crypto/ssh/knownhosts"
|
||||||
|
)
|
||||||
|
|
||||||
|
// SSHOptions is the SSH connection and authentication surface shared by
|
||||||
|
// the deploy command and the source installer. Secrets (passwords,
|
||||||
|
// passphrases, sudo passwords) can be supplied through direct flags or
|
||||||
|
// file options. The CLI layer should strongly prefer file options: when
|
||||||
|
// read from a file they never appear in command arguments, logs, or
|
||||||
|
// shell history, while direct flags expose the value through the process
|
||||||
|
// list and shell history.
|
||||||
|
type SSHOptions struct {
|
||||||
|
Host string
|
||||||
|
Port int
|
||||||
|
User string
|
||||||
|
IdentityFile string
|
||||||
|
KeyPassphrase string
|
||||||
|
Password string
|
||||||
|
SudoPassword string
|
||||||
|
KnownHostsFile string
|
||||||
|
HostKeyFingerprint string
|
||||||
|
InsecureHostKey bool
|
||||||
|
}
|
||||||
|
|
||||||
|
type DeployOptions struct {
|
||||||
|
SSHOptions
|
||||||
|
Binary string
|
||||||
|
Token string
|
||||||
|
URL string
|
||||||
|
Docker bool
|
||||||
|
Image string
|
||||||
|
NoStart bool
|
||||||
|
}
|
||||||
|
|
||||||
|
func Deploy(opts DeployOptions) error {
|
||||||
|
if opts.Host == "" || opts.User == "" {
|
||||||
|
return errors.New("--host and --user are required")
|
||||||
|
}
|
||||||
|
if opts.Port == 0 {
|
||||||
|
opts.Port = 22
|
||||||
|
}
|
||||||
|
if opts.Port < 1 || opts.Port > 65535 {
|
||||||
|
return errors.New("SSH port must be between 1 and 65535")
|
||||||
|
}
|
||||||
|
if err := ValidateToken(opts.Token); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if opts.URL == "" {
|
||||||
|
opts.URL = DefaultURL
|
||||||
|
}
|
||||||
|
if opts.Image == "" {
|
||||||
|
opts.Image = DefaultImage
|
||||||
|
}
|
||||||
|
if opts.Docker {
|
||||||
|
if err := ValidateImage(opts.Image); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := ValidateURL(opts.URL); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if !opts.Docker && opts.Binary == "" {
|
||||||
|
var err error
|
||||||
|
opts.Binary, err = os.Executable()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
auth, err := sshAuth(opts.SSHOptions)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
hostKey, err := hostKeyCallback(opts.SSHOptions)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
client, err := ssh.Dial("tcp", net.JoinHostPort(opts.Host, strconv.Itoa(opts.Port)), &ssh.ClientConfig{
|
||||||
|
User: opts.User,
|
||||||
|
Auth: auth,
|
||||||
|
HostKeyCallback: hostKey,
|
||||||
|
Timeout: 15 * time.Second,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("connect to %s: %w", opts.Host, err)
|
||||||
|
}
|
||||||
|
defer client.Close()
|
||||||
|
|
||||||
|
suffix := strconv.FormatInt(time.Now().UnixNano(), 36)
|
||||||
|
remoteBinary := "/tmp/rsmon-worker-" + suffix
|
||||||
|
remoteEnv := remoteBinary + ".env"
|
||||||
|
remoteUnit := remoteBinary + ".service"
|
||||||
|
if err := uploadBytes(client, Environment(opts.URL, opts.Token), remoteEnv, 0600); err != nil {
|
||||||
|
return fmt.Errorf("upload configuration: %w", err)
|
||||||
|
}
|
||||||
|
defer runRemote(client, "rm -f -- "+shellQuote(remoteBinary)+" "+shellQuote(remoteEnv)+" "+shellQuote(remoteUnit), nil) //nolint:errcheck
|
||||||
|
if opts.Docker {
|
||||||
|
if err := uploadBytes(client, []byte(DockerUnit(opts.Image)), remoteUnit, 0600); err != nil {
|
||||||
|
return fmt.Errorf("upload systemd unit: %w", err)
|
||||||
|
}
|
||||||
|
return deployDocker(client, opts, remoteEnv, remoteUnit)
|
||||||
|
}
|
||||||
|
if err := upload(client, opts.Binary, remoteBinary, 0700); err != nil {
|
||||||
|
return fmt.Errorf("upload worker: %w", err)
|
||||||
|
}
|
||||||
|
args := shellQuote(remoteBinary) + " install --binary " + shellQuote(remoteBinary) + " --env-file " + shellQuote(remoteEnv)
|
||||||
|
if opts.NoStart {
|
||||||
|
args += " --no-start"
|
||||||
|
}
|
||||||
|
command, stdin := sudoWrap(opts.User, opts.SudoPassword, args)
|
||||||
|
if err := runRemote(client, command, stdin); err != nil {
|
||||||
|
return fmt.Errorf("remote install: %w", err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func deployDocker(client *ssh.Client, opts DeployOptions, remoteEnv, remoteUnit string) error {
|
||||||
|
script := "install -d -m 0755 /etc/rsmon-worker" +
|
||||||
|
" && install -m 0600 " + shellQuote(remoteEnv) + " /etc/rsmon-worker/worker.env" +
|
||||||
|
" && install -m 0644 " + shellQuote(remoteUnit) + " /etc/systemd/system/rsmon-worker.service" +
|
||||||
|
" && docker pull " + shellQuote(opts.Image) +
|
||||||
|
" && systemctl daemon-reload" +
|
||||||
|
" && systemctl enable rsmon-worker.service"
|
||||||
|
if !opts.NoStart {
|
||||||
|
script += " && systemctl restart rsmon-worker.service && systemctl is-active --quiet rsmon-worker.service"
|
||||||
|
}
|
||||||
|
command, stdin := sudoWrap(opts.User, opts.SudoPassword, "sh -c "+shellQuote(script))
|
||||||
|
if err := runRemote(client, command, stdin); err != nil {
|
||||||
|
return fmt.Errorf("remote Docker install: %w", err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// sudoWrap prefixes a remote command with the privilege path required to
|
||||||
|
// run it as root: the plain command for a root user, `sudo -n` when the
|
||||||
|
// user has passwordless sudo, and `sudo -S` with an empty prompt when a
|
||||||
|
// sudo password is configured. The sudo password is delivered only over
|
||||||
|
// the session's stdin, never in the command string, so it cannot appear
|
||||||
|
// in process listings, logs, or shell history.
|
||||||
|
func sudoWrap(user, sudoPassword, command string) (string, []byte) {
|
||||||
|
if user == "root" {
|
||||||
|
return command, nil
|
||||||
|
}
|
||||||
|
if sudoPassword != "" {
|
||||||
|
return "sudo -S -p '' -- " + command, []byte(sudoPassword + "\n")
|
||||||
|
}
|
||||||
|
return "sudo -n -- " + command, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func sshAuth(opts SSHOptions) ([]ssh.AuthMethod, error) {
|
||||||
|
var methods []ssh.AuthMethod
|
||||||
|
if opts.IdentityFile != "" {
|
||||||
|
key, err := os.ReadFile(opts.IdentityFile)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("read identity file: %w", err)
|
||||||
|
}
|
||||||
|
var signer ssh.Signer
|
||||||
|
if opts.KeyPassphrase != "" {
|
||||||
|
signer, err = ssh.ParsePrivateKeyWithPassphrase(key, []byte(opts.KeyPassphrase))
|
||||||
|
} else {
|
||||||
|
signer, err = ssh.ParsePrivateKey(key)
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("parse identity file: %w", err)
|
||||||
|
}
|
||||||
|
methods = append(methods, ssh.PublicKeys(signer))
|
||||||
|
}
|
||||||
|
if opts.Password != "" {
|
||||||
|
methods = append(methods, ssh.Password(opts.Password))
|
||||||
|
}
|
||||||
|
if len(methods) == 0 {
|
||||||
|
return nil, errors.New("provide --identity-file or --password")
|
||||||
|
}
|
||||||
|
return methods, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func hostKeyCallback(opts SSHOptions) (ssh.HostKeyCallback, error) {
|
||||||
|
if opts.HostKeyFingerprint != "" {
|
||||||
|
want := opts.HostKeyFingerprint
|
||||||
|
return func(_ string, _ net.Addr, key ssh.PublicKey) error {
|
||||||
|
if got := ssh.FingerprintSHA256(key); got != want {
|
||||||
|
return fmt.Errorf("host key fingerprint mismatch: got %s", got)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
if opts.InsecureHostKey {
|
||||||
|
return ssh.InsecureIgnoreHostKey(), nil //nolint:gosec // explicit operator opt-in
|
||||||
|
}
|
||||||
|
path := opts.KnownHostsFile
|
||||||
|
if path == "" {
|
||||||
|
u, err := user.Current()
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
path = filepath.Join(u.HomeDir, ".ssh", "known_hosts")
|
||||||
|
}
|
||||||
|
return knownhosts.New(path)
|
||||||
|
}
|
||||||
|
|
||||||
|
func upload(client *ssh.Client, localPath, remotePath string, mode os.FileMode) error {
|
||||||
|
f, err := os.Open(localPath)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer f.Close()
|
||||||
|
return uploadReader(client, f, remotePath, mode)
|
||||||
|
}
|
||||||
|
|
||||||
|
func uploadBytes(client *ssh.Client, data []byte, remotePath string, mode os.FileMode) error {
|
||||||
|
return uploadReader(client, bytes.NewReader(data), remotePath, mode)
|
||||||
|
}
|
||||||
|
|
||||||
|
func uploadReader(client *ssh.Client, src io.Reader, remotePath string, mode os.FileMode) error {
|
||||||
|
session, err := client.NewSession()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer session.Close()
|
||||||
|
stdin, err := session.StdinPipe()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
session.Stdout = os.Stdout
|
||||||
|
session.Stderr = os.Stderr
|
||||||
|
command := "umask 077; base64 -d > " + shellQuote(remotePath) + " && chmod " + fmt.Sprintf("%#o", mode.Perm()) + " " + shellQuote(remotePath)
|
||||||
|
if err := session.Start(command); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
encoder := base64.NewEncoder(base64.StdEncoding, stdin)
|
||||||
|
_, copyErr := io.Copy(encoder, src)
|
||||||
|
closeErr := encoder.Close()
|
||||||
|
pipeErr := stdin.Close()
|
||||||
|
waitErr := session.Wait()
|
||||||
|
return errors.Join(copyErr, closeErr, pipeErr, waitErr)
|
||||||
|
}
|
||||||
|
|
||||||
|
func runRemote(client *ssh.Client, command string, stdin []byte) error {
|
||||||
|
// Deploy commands are short and stream to the terminal; they keep the
|
||||||
|
// historical behavior with no timeout.
|
||||||
|
return runRemoteWithTimeout(client, command, stdin, os.Stdout, os.Stderr, 0)
|
||||||
|
}
|
||||||
|
|
||||||
|
// maxRemoteError bounds the stderr snippet folded into runRemoteOutput
|
||||||
|
// errors so a verbose remote failure cannot produce an unbounded error
|
||||||
|
// string.
|
||||||
|
const maxRemoteError = 4096
|
||||||
|
|
||||||
|
// maxRemoteOutput bounds the stdout captured by runRemoteOutput so a
|
||||||
|
// noisy remote command cannot exhaust memory.
|
||||||
|
const maxRemoteOutput = 1 << 20 // 1 MiB
|
||||||
|
|
||||||
|
// boundedBuffer is an io.Writer that silently discards everything past
|
||||||
|
// max bytes and remembers whether truncation happened.
|
||||||
|
type boundedBuffer struct {
|
||||||
|
buf bytes.Buffer
|
||||||
|
max int
|
||||||
|
truncated bool
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b *boundedBuffer) Write(p []byte) (int, error) {
|
||||||
|
if len(p) == 0 {
|
||||||
|
return 0, nil
|
||||||
|
}
|
||||||
|
if b.buf.Len() >= b.max {
|
||||||
|
b.truncated = true
|
||||||
|
return len(p), nil
|
||||||
|
}
|
||||||
|
remaining := b.max - b.buf.Len()
|
||||||
|
if len(p) > remaining {
|
||||||
|
b.buf.Write(p[:remaining])
|
||||||
|
b.truncated = true
|
||||||
|
return len(p), nil
|
||||||
|
}
|
||||||
|
return b.buf.Write(p)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b *boundedBuffer) Bytes() []byte { return b.buf.Bytes() }
|
||||||
|
|
||||||
|
func (b *boundedBuffer) String() string { return b.buf.String() }
|
||||||
|
|
||||||
|
// runRemoteOutput executes a remote command and returns its captured,
|
||||||
|
// size-bounded stdout. Stderr is folded into the returned error on
|
||||||
|
// failure (bounded to maxRemoteError bytes) so operators see what went
|
||||||
|
// wrong without a bounded failure dumping unbounded output. The command
|
||||||
|
// is aborted if it outlives timeout (<= 0 disables the timeout).
|
||||||
|
func runRemoteOutput(client *ssh.Client, command string, stdin []byte, timeout time.Duration) ([]byte, error) {
|
||||||
|
var stdout, stderr boundedBuffer
|
||||||
|
stdout.max = maxRemoteOutput
|
||||||
|
stderr.max = maxRemoteError
|
||||||
|
if err := runRemoteWithTimeout(client, command, stdin, &stdout, &stderr, timeout); err != nil {
|
||||||
|
msg := strings.TrimSpace(stderr.String())
|
||||||
|
if msg != "" {
|
||||||
|
if stderr.truncated {
|
||||||
|
msg += "..."
|
||||||
|
}
|
||||||
|
return stdout.Bytes(), fmt.Errorf("%w: %s", err, msg)
|
||||||
|
}
|
||||||
|
return stdout.Bytes(), err
|
||||||
|
}
|
||||||
|
out := stdout.Bytes()
|
||||||
|
if stdout.truncated {
|
||||||
|
out = append(out, []byte("\n...[output truncated]")...)
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// runRemoteWithTimeout runs a remote command, optionally aborting it
|
||||||
|
// when it outlives timeout (<= 0 disables the timeout). The session is
|
||||||
|
// closed and the blocked Run is unblocked when the timer fires.
|
||||||
|
func runRemoteWithTimeout(client *ssh.Client, command string, stdin []byte, stdout, stderr io.Writer, timeout time.Duration) error {
|
||||||
|
session, err := client.NewSession()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer session.Close()
|
||||||
|
if stdin != nil {
|
||||||
|
session.Stdin = bytes.NewReader(stdin)
|
||||||
|
}
|
||||||
|
session.Stdout = stdout
|
||||||
|
session.Stderr = stderr
|
||||||
|
|
||||||
|
if timeout <= 0 {
|
||||||
|
return session.Run(command)
|
||||||
|
}
|
||||||
|
done := make(chan error, 1)
|
||||||
|
go func() { done <- session.Run(command) }()
|
||||||
|
timer := time.NewTimer(timeout)
|
||||||
|
defer timer.Stop()
|
||||||
|
select {
|
||||||
|
case err := <-done:
|
||||||
|
return err
|
||||||
|
case <-timer.C:
|
||||||
|
_ = session.Close() // abort the remote command and free the session
|
||||||
|
<-done
|
||||||
|
return fmt.Errorf("remote command timed out after %s", timeout)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func shellQuote(value string) string {
|
||||||
|
return "'" + strings.ReplaceAll(value, "'", "'\\''") + "'"
|
||||||
|
}
|
||||||
57
internal/installer/deploy_test.go
Обычный файл
57
internal/installer/deploy_test.go
Обычный файл
@@ -0,0 +1,57 @@
|
|||||||
|
package installer
|
||||||
|
|
||||||
|
import (
|
||||||
|
"crypto/ed25519"
|
||||||
|
"crypto/rand"
|
||||||
|
"net"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"golang.org/x/crypto/ssh"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestFingerprintHostKeyCallback(t *testing.T) {
|
||||||
|
_, privateKey, err := ed25519.GenerateKey(rand.Reader)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
publicKey, err := ssh.NewPublicKey(privateKey.Public())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
callback, err := hostKeyCallback(SSHOptions{HostKeyFingerprint: ssh.FingerprintSHA256(publicKey)})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := callback("host", &net.TCPAddr{}, publicKey); err != nil {
|
||||||
|
t.Fatalf("matching fingerprint rejected: %v", err)
|
||||||
|
}
|
||||||
|
callback, err = hostKeyCallback(SSHOptions{HostKeyFingerprint: "SHA256:wrong"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := callback("host", &net.TCPAddr{}, publicKey); err == nil {
|
||||||
|
t.Fatal("mismatched fingerprint accepted")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestKnownHostsMissingFile(t *testing.T) {
|
||||||
|
if _, err := hostKeyCallback(SSHOptions{KnownHostsFile: t.TempDir() + "/missing"}); err == nil {
|
||||||
|
t.Fatal("missing known_hosts file accepted")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDeployRejectsMutableDockerImageBeforeConnecting(t *testing.T) {
|
||||||
|
err := Deploy(DeployOptions{
|
||||||
|
SSHOptions: SSHOptions{
|
||||||
|
Host: "unreachable.example.test",
|
||||||
|
User: "deploy",
|
||||||
|
},
|
||||||
|
Token: "token",
|
||||||
|
Docker: true,
|
||||||
|
Image: "reg.rsxx.ru/rsmon/rsmon-worker:latest",
|
||||||
|
})
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "immutable") {
|
||||||
|
t.Fatalf("Deploy() error = %v, want immutable image error", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
621
internal/installer/harness/harness.go
Обычный файл
621
internal/installer/harness/harness.go
Обычный файл
@@ -0,0 +1,621 @@
|
|||||||
|
// Package harness starts real OpenSSH containers for the source-install
|
||||||
|
// tests. It is the reusable Docker/OpenSSH test harness from work
|
||||||
|
// package 1 of docs/source-installation.md.
|
||||||
|
//
|
||||||
|
// The harness never mocks SSH: it builds a distro fixture image, runs an
|
||||||
|
// OpenSSH server in a container, waits for real network readiness, and
|
||||||
|
// connects with the golang.org/x/crypto/ssh library and known_hosts
|
||||||
|
// verification semantics the worker installer uses. Tests opt in with
|
||||||
|
// RSMON_TEST_DOCKER=1 so ordinary unit runs never pull or start Docker.
|
||||||
|
package harness
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"net"
|
||||||
|
"os"
|
||||||
|
"os/exec"
|
||||||
|
"path/filepath"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"golang.org/x/crypto/ssh"
|
||||||
|
"golang.org/x/crypto/ssh/knownhosts"
|
||||||
|
|
||||||
|
"rocketgit.ru/rsmon/worker/internal/sshinstall"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Fixture describes one distro OpenSSH fixture the harness can start.
|
||||||
|
type Fixture struct {
|
||||||
|
Name string // "alpine", "ubuntu", "arch"
|
||||||
|
Distro sshinstall.Distro // expected detected distro
|
||||||
|
Pkg sshinstall.PackageManager
|
||||||
|
Init sshinstall.InitSystem
|
||||||
|
Image string // explicit image override; empty uses the default
|
||||||
|
User string // SSH user to authenticate as; default "root"
|
||||||
|
}
|
||||||
|
|
||||||
|
// Fixtures returns the supported source-install distro fixtures.
|
||||||
|
func Fixtures() []Fixture {
|
||||||
|
return []Fixture{
|
||||||
|
{Name: "alpine", Distro: sshinstall.DistroAlpine, Pkg: sshinstall.PkgApk, Init: sshinstall.InitOpenRC},
|
||||||
|
{Name: "ubuntu", Distro: sshinstall.DistroUbuntu, Pkg: sshinstall.PkgApt, Init: sshinstall.InitSystemd},
|
||||||
|
{Name: "arch", Distro: sshinstall.DistroArch, Pkg: sshinstall.PkgPacman, Init: sshinstall.InitSystemd},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// defaultImages maps fixture name to its base image. reg.rsxx.ru mirror
|
||||||
|
// refs are used where the mirror caches the distro; Docker Hub refs are
|
||||||
|
// the fallback for distros the mirror does not carry and are overridable
|
||||||
|
// per fixture via RSMON_TEST_IMAGE_<NAME>.
|
||||||
|
var defaultImages = map[string]string{
|
||||||
|
"alpine": "reg.rsxx.ru/library/alpine:3",
|
||||||
|
"ubuntu": "ubuntu:24.04",
|
||||||
|
"arch": "archlinux:latest",
|
||||||
|
}
|
||||||
|
|
||||||
|
// ImageRef resolves the base image reference for a fixture. Precedence:
|
||||||
|
// RSMON_TEST_IMAGE_<NAME> env > Fixture.Image > mirror-aware default.
|
||||||
|
func (f Fixture) ImageRef() string {
|
||||||
|
envKey := "RSMON_TEST_IMAGE_" + strings.ToUpper(strings.ReplaceAll(f.Name, "-", "_"))
|
||||||
|
if v := strings.TrimSpace(os.Getenv(envKey)); v != "" {
|
||||||
|
return v
|
||||||
|
}
|
||||||
|
if f.Image != "" {
|
||||||
|
return f.Image
|
||||||
|
}
|
||||||
|
return defaultImages[f.Name]
|
||||||
|
}
|
||||||
|
|
||||||
|
// UserOrDefault returns the SSH user, defaulting to root.
|
||||||
|
func (f Fixture) UserOrDefault() string {
|
||||||
|
if f.User != "" {
|
||||||
|
return f.User
|
||||||
|
}
|
||||||
|
return "root"
|
||||||
|
}
|
||||||
|
|
||||||
|
// DockerfilePath is the fixture's Dockerfile under the package testdata.
|
||||||
|
func (f Fixture) DockerfilePath() string {
|
||||||
|
return filepath.Join("testdata", "fixtures", f.Name, "Dockerfile")
|
||||||
|
}
|
||||||
|
|
||||||
|
// BuildContext is the Docker build context that carries the shared keys.
|
||||||
|
func (f Fixture) BuildContext() string {
|
||||||
|
return filepath.Join("testdata")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Harness manages the lifecycle of one OpenSSH test container: build,
|
||||||
|
// run, wait for readiness, capture the host key, and tear down reliably.
|
||||||
|
// Every resource it creates - fixture image tag, container, network, and
|
||||||
|
// the temp known_hosts directory - is unique to this instance and is
|
||||||
|
// removed by Stop.
|
||||||
|
type Harness struct {
|
||||||
|
Name string
|
||||||
|
Fixture Fixture
|
||||||
|
|
||||||
|
mu sync.Mutex
|
||||||
|
started bool
|
||||||
|
suffix string
|
||||||
|
containerID string
|
||||||
|
container string
|
||||||
|
network string
|
||||||
|
imageTag string
|
||||||
|
port int
|
||||||
|
user string
|
||||||
|
keyPath string
|
||||||
|
knownHosts string
|
||||||
|
}
|
||||||
|
|
||||||
|
// New creates a harness for the named fixture. The name must be a safe,
|
||||||
|
// short identifier; a random suffix makes the container, network, and
|
||||||
|
// fixture-image tag unique to this instance so teardown never touches
|
||||||
|
// another harness's resources.
|
||||||
|
func New(name string, f Fixture) (*Harness, error) {
|
||||||
|
if err := validateName(name); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
suffix := strconv.FormatInt(time.Now().UnixNano(), 36)
|
||||||
|
return &Harness{
|
||||||
|
Name: name,
|
||||||
|
Fixture: f,
|
||||||
|
suffix: suffix,
|
||||||
|
container: "rsmon-worker-test-" + sanitize(name) + "-" + suffix,
|
||||||
|
network: "rsmon-worker-test-" + sanitize(name) + "-" + suffix,
|
||||||
|
imageTag: "rsmon-worker-test/" + sanitize(f.Name) + "-" + suffix + ":local",
|
||||||
|
user: f.UserOrDefault(),
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// ContainerID returns the running container id (after Start).
|
||||||
|
func (h *Harness) ContainerID() string {
|
||||||
|
h.mu.Lock()
|
||||||
|
defer h.mu.Unlock()
|
||||||
|
return h.containerID
|
||||||
|
}
|
||||||
|
|
||||||
|
// ContainerName returns the Docker container name.
|
||||||
|
func (h *Harness) ContainerName() string { return h.container }
|
||||||
|
|
||||||
|
// NetworkName returns the Docker network name.
|
||||||
|
func (h *Harness) NetworkName() string { return h.network }
|
||||||
|
|
||||||
|
// ImageTag returns the fixture image tag this harness instance builds
|
||||||
|
// and removes on Stop. Tags are unique per instance, so removing one
|
||||||
|
// never deletes a shared base image or another harness's image.
|
||||||
|
func (h *Harness) ImageTag() string { return h.imageTag }
|
||||||
|
|
||||||
|
// KnownHostsPath returns the temp known_hosts file created by Start, or
|
||||||
|
// "" before Start. The surrounding directory is removed by Stop.
|
||||||
|
func (h *Harness) KnownHostsPath() string {
|
||||||
|
h.mu.Lock()
|
||||||
|
defer h.mu.Unlock()
|
||||||
|
return h.knownHosts
|
||||||
|
}
|
||||||
|
|
||||||
|
// Addr returns the dialable host:port of the published SSH listener.
|
||||||
|
// Before Start it is empty.
|
||||||
|
func (h *Harness) Addr() string {
|
||||||
|
h.mu.Lock()
|
||||||
|
defer h.mu.Unlock()
|
||||||
|
if h.port == 0 {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return net.JoinHostPort("127.0.0.1", strconv.Itoa(h.port))
|
||||||
|
}
|
||||||
|
|
||||||
|
// Port returns the published host port of the container's SSH listener
|
||||||
|
// (the host is always 127.0.0.1). 0 before Start.
|
||||||
|
func (h *Harness) Port() int {
|
||||||
|
h.mu.Lock()
|
||||||
|
defer h.mu.Unlock()
|
||||||
|
return h.port
|
||||||
|
}
|
||||||
|
|
||||||
|
// Start builds the fixture image, starts the container, waits for real
|
||||||
|
// SSH readiness, captures the server host key into a temp known_hosts
|
||||||
|
// file, and records the published port. Every error path cleans up the
|
||||||
|
// container, network, image tag, and temp dir created so far; on
|
||||||
|
// success the caller owns teardown via t.Cleanup(h.Stop) or a defer.
|
||||||
|
func (h *Harness) Start(ctx context.Context) error {
|
||||||
|
if !Enabled() {
|
||||||
|
return errors.New("harness integration is disabled; set RSMON_TEST_DOCKER=1 to run Docker/OpenSSH tests")
|
||||||
|
}
|
||||||
|
image := h.Fixture.ImageRef()
|
||||||
|
if image == "" {
|
||||||
|
return fmt.Errorf("fixture %q has no base image", h.Fixture.Name)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Any error after the first resource is created must release what
|
||||||
|
// was already allocated. Stop is idempotent and tolerates resources
|
||||||
|
// that were never created.
|
||||||
|
success := false
|
||||||
|
defer func() {
|
||||||
|
if !success {
|
||||||
|
_ = h.Stop()
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
if _, err := dockerCmd(ctx, "build", "-q", "-t", h.imageTag, "-f", h.Fixture.DockerfilePath(), h.Fixture.BuildContext()); err != nil {
|
||||||
|
return fmt.Errorf("build %s fixture image: %w", h.Fixture.Name, err)
|
||||||
|
}
|
||||||
|
if _, err := dockerCmd(ctx, "network", "create", h.network); err != nil {
|
||||||
|
return fmt.Errorf("create test network: %w", err)
|
||||||
|
}
|
||||||
|
// docker run -d prints the container id directly, so no lookup is
|
||||||
|
// needed; the container name is the stable handle for later docker
|
||||||
|
// calls and the id is captured for diagnostics and assertions.
|
||||||
|
runArgs := []string{"run", "-d", "--name", h.container, "--network", h.network, "-p", "127.0.0.1::22"}
|
||||||
|
runArgs = append(runArgs, dockerDNS()...)
|
||||||
|
runArgs = append(runArgs, h.imageTag)
|
||||||
|
out, err := dockerCmd(ctx, runArgs...)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("start %s fixture container: %w", h.Fixture.Name, err)
|
||||||
|
}
|
||||||
|
h.containerID = strings.TrimSpace(out)
|
||||||
|
|
||||||
|
port, err := h.publishedPort(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
h.port = port
|
||||||
|
|
||||||
|
key, err := h.waitForSSH(ctx, h.Addr())
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("wait for ssh readiness on %s fixture: %w", h.Fixture.Name, err)
|
||||||
|
}
|
||||||
|
// Register the temp dir before writing so a write failure still
|
||||||
|
// leaves it known to the cleanup defer.
|
||||||
|
dir, err := os.MkdirTemp("", "rsmon-worker-harness-")
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
h.knownHosts = filepath.Join(dir, "known_hosts")
|
||||||
|
hostsFile, err := writeKnownHosts(dir, h.Addr(), key)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
h.keyPath = testKeyPath()
|
||||||
|
h.knownHosts = hostsFile
|
||||||
|
|
||||||
|
h.mu.Lock()
|
||||||
|
h.started = true
|
||||||
|
h.mu.Unlock()
|
||||||
|
success = true
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// dockerDNS returns the `--dns` arguments to pin for fixture containers,
|
||||||
|
// parsed from the comma-separated RSMON_TEST_DOCKER_DNS environment
|
||||||
|
// variable. It is empty by default (Docker's embedded DNS). The override
|
||||||
|
// exists so environments with flaky local resolvers can pin a reliable
|
||||||
|
// upstream for the internet-facing installs (go.dev, rocketgit.ru,
|
||||||
|
// proxy.golang.org), which would otherwise fail intermittently on DNS
|
||||||
|
// timeouts.
|
||||||
|
func dockerDNS() []string {
|
||||||
|
var args []string
|
||||||
|
for _, ns := range strings.Split(os.Getenv("RSMON_TEST_DOCKER_DNS"), ",") {
|
||||||
|
if ns = strings.TrimSpace(ns); ns != "" {
|
||||||
|
args = append(args, "--dns", ns)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return args
|
||||||
|
}
|
||||||
|
|
||||||
|
// Stop releases every resource the harness created: the container, its
|
||||||
|
// dedicated network, the per-instance fixture image tag (never a shared
|
||||||
|
// base image), and the temp known_hosts directory. It is idempotent and
|
||||||
|
// tolerates already-removed resources so teardown never fails the test
|
||||||
|
// twice.
|
||||||
|
func (h *Harness) Stop() error {
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
|
||||||
|
defer cancel()
|
||||||
|
var errs []error
|
||||||
|
if h.container != "" {
|
||||||
|
if _, err := dockerCmd(ctx, "rm", "-f", h.container); err != nil && !isNotExist(err) {
|
||||||
|
errs = append(errs, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if h.network != "" {
|
||||||
|
if _, err := dockerCmd(ctx, "network", "rm", h.network); err != nil && !isNotExist(err) {
|
||||||
|
errs = append(errs, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if h.imageTag != "" {
|
||||||
|
if _, err := dockerCmd(ctx, "image", "rm", h.imageTag); err != nil && !isNotExist(err) && !isInUse(err) {
|
||||||
|
errs = append(errs, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if h.knownHosts != "" {
|
||||||
|
if err := os.RemoveAll(filepath.Dir(h.knownHosts)); err != nil {
|
||||||
|
errs = append(errs, fmt.Errorf("remove harness temp dir: %w", err))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
h.mu.Lock()
|
||||||
|
h.started = false
|
||||||
|
h.containerID = ""
|
||||||
|
h.port = 0
|
||||||
|
h.keyPath = ""
|
||||||
|
h.knownHosts = ""
|
||||||
|
h.mu.Unlock()
|
||||||
|
return errors.Join(errs...)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Dial connects using the same SSH library (golang.org/x/crypto/ssh) and
|
||||||
|
// the same known_hosts host-key verification the installer's deploy path
|
||||||
|
// relies on. The harness owns this code rather than calling into the
|
||||||
|
// installer package so tests stay independent; only the library and the
|
||||||
|
// verification semantics are shared.
|
||||||
|
func (h *Harness) Dial() (*ssh.Client, error) {
|
||||||
|
h.mu.Lock()
|
||||||
|
if !h.started || h.knownHosts == "" || h.keyPath == "" {
|
||||||
|
h.mu.Unlock()
|
||||||
|
return nil, errors.New("harness is not started")
|
||||||
|
}
|
||||||
|
addr := net.JoinHostPort("127.0.0.1", strconv.Itoa(h.port))
|
||||||
|
knownHostsFile, keyPath, user := h.knownHosts, h.keyPath, h.user
|
||||||
|
h.mu.Unlock()
|
||||||
|
|
||||||
|
keyBytes, err := os.ReadFile(keyPath)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
signer, err := ssh.ParsePrivateKey(keyBytes)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
hostKeyCallback, err := knownhosts.New(knownHostsFile)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
config := &ssh.ClientConfig{
|
||||||
|
User: user,
|
||||||
|
Auth: []ssh.AuthMethod{ssh.PublicKeys(signer)},
|
||||||
|
HostKeyCallback: hostKeyCallback,
|
||||||
|
Timeout: 15 * time.Second,
|
||||||
|
}
|
||||||
|
return ssh.Dial("tcp", addr, config)
|
||||||
|
}
|
||||||
|
|
||||||
|
// RunCommand executes a command over an existing SSH client and returns
|
||||||
|
// its stdout. Stderr is folded into the returned error message on
|
||||||
|
// failure so test failures show what went wrong.
|
||||||
|
func RunCommand(client *ssh.Client, command string) ([]byte, error) {
|
||||||
|
session, err := client.NewSession()
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer session.Close() //nolint:errcheck
|
||||||
|
var stdout, stderr bytes.Buffer
|
||||||
|
session.Stdout = &stdout
|
||||||
|
session.Stderr = &stderr
|
||||||
|
if err := session.Run(command); err != nil {
|
||||||
|
if msg := strings.TrimSpace(stderr.String()); msg != "" {
|
||||||
|
return stdout.Bytes(), fmt.Errorf("%w: %s", err, msg)
|
||||||
|
}
|
||||||
|
return stdout.Bytes(), err
|
||||||
|
}
|
||||||
|
return stdout.Bytes(), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// DockerCmd runs the Docker CLI with the given arguments, returning
|
||||||
|
// stdout. Exposed so integration tests can assert teardown state.
|
||||||
|
func (h *Harness) DockerCmd(ctx context.Context, args ...string) (string, error) {
|
||||||
|
return dockerCmd(ctx, args...)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Enabled reports whether Docker-backed integration tests may run. The
|
||||||
|
// opt-in env var keeps ordinary `go test` and CI unit runs from pulling
|
||||||
|
// or starting any container.
|
||||||
|
func Enabled() bool {
|
||||||
|
switch strings.ToLower(strings.TrimSpace(os.Getenv("RSMON_TEST_DOCKER"))) {
|
||||||
|
case "1", "true", "yes", "on":
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// SkipUnlessEnabled skips the test with a helpful message unless the
|
||||||
|
// Docker integration opt-in is set.
|
||||||
|
func SkipUnlessEnabled(t interface {
|
||||||
|
Helper()
|
||||||
|
Skipf(format string, args ...interface{})
|
||||||
|
},
|
||||||
|
) {
|
||||||
|
t.Helper()
|
||||||
|
if !Enabled() {
|
||||||
|
t.Skipf("Docker/OpenSSH harness tests are opt-in; set RSMON_TEST_DOCKER=1 to run them")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// dockerBin is the Docker CLI binary. Overridable in tests via
|
||||||
|
// SetDockerBin so unit tests can stub the harness's docker calls.
|
||||||
|
var dockerBin = "docker"
|
||||||
|
|
||||||
|
// SetDockerBin overrides the Docker CLI binary used by the harness.
|
||||||
|
// Pass an empty value to restore the default ("docker").
|
||||||
|
func SetDockerBin(name string) {
|
||||||
|
if name == "" {
|
||||||
|
dockerBin = "docker"
|
||||||
|
return
|
||||||
|
}
|
||||||
|
dockerBin = name
|
||||||
|
}
|
||||||
|
|
||||||
|
func dockerCmd(ctx context.Context, args ...string) (string, error) {
|
||||||
|
cmd := exec.CommandContext(ctx, dockerBin, args...)
|
||||||
|
out, err := cmd.Output()
|
||||||
|
if err != nil {
|
||||||
|
if ee, ok := err.(*exec.ExitError); ok && len(ee.Stderr) > 0 {
|
||||||
|
return "", fmt.Errorf("docker %s: %s", strings.Join(args, " "), strings.TrimSpace(string(ee.Stderr)))
|
||||||
|
}
|
||||||
|
return "", fmt.Errorf("docker %s: %w", strings.Join(args, " "), err)
|
||||||
|
}
|
||||||
|
return string(out), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// isNotExist reports whether the docker error is a missing resource
|
||||||
|
// (container/network/image already removed or never created). Docker
|
||||||
|
// reports missing networks and images as "... not found" and missing
|
||||||
|
// containers as "No such container: ...".
|
||||||
|
func isNotExist(err error) bool {
|
||||||
|
if err == nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
lower := strings.ToLower(err.Error())
|
||||||
|
for _, marker := range []string{"no such container", "no such network", "no such image", "is not running", "not found"} {
|
||||||
|
if strings.Contains(lower, marker) {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// isInUse reports whether a docker image removal failed because another
|
||||||
|
// container or tag still references the image. Teardown must not treat
|
||||||
|
// that as a leak error: the per-instance tags make this unlikely, but
|
||||||
|
// tolerating it keeps Stop deterministic under concurrent harnesses.
|
||||||
|
func isInUse(err error) bool {
|
||||||
|
if err == nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
lower := strings.ToLower(err.Error())
|
||||||
|
for _, marker := range []string{"image is being used", "image is in use", "image is referenced"} {
|
||||||
|
if strings.Contains(lower, marker) {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// publishedPort queries the host port docker published for the
|
||||||
|
// container's port 22.
|
||||||
|
func (h *Harness) publishedPort(ctx context.Context) (int, error) {
|
||||||
|
out, err := dockerCmd(ctx, "port", h.container, "22/tcp")
|
||||||
|
if err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
return parsePublishedPort(out)
|
||||||
|
}
|
||||||
|
|
||||||
|
// parsePublishedPort extracts the host port from `docker port` output
|
||||||
|
// like "127.0.0.1:49153", "0.0.0.0:49153", or "::1:49153".
|
||||||
|
func parsePublishedPort(out string) (int, error) {
|
||||||
|
for _, line := range strings.Split(out, "\n") {
|
||||||
|
line = strings.TrimSpace(line)
|
||||||
|
if line == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
sep := strings.LastIndex(line, ":")
|
||||||
|
if sep < 0 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
port, err := strconv.Atoi(strings.TrimSpace(line[sep+1:]))
|
||||||
|
if err == nil && port > 0 && port <= 65535 {
|
||||||
|
return port, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return 0, fmt.Errorf("could not parse published port from %q", out)
|
||||||
|
}
|
||||||
|
|
||||||
|
// waitForSSH waits for the container's SSH listener to accept a real
|
||||||
|
// handshake and returns the server host key. TCP readiness alone is not
|
||||||
|
// enough; the ssh.Dial must succeed and the exec channel must answer.
|
||||||
|
func (h *Harness) waitForSSH(ctx context.Context, addr string) (ssh.PublicKey, error) {
|
||||||
|
if err := waitForPort(ctx, addr, 60*time.Second); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
deadline := time.Now().Add(60 * time.Second)
|
||||||
|
var lastErr error
|
||||||
|
for {
|
||||||
|
if ctx.Err() != nil {
|
||||||
|
return nil, fmt.Errorf("ssh readiness: %w (last: %v)", ctx.Err(), lastErr)
|
||||||
|
}
|
||||||
|
if time.Now().After(deadline) {
|
||||||
|
return nil, fmt.Errorf("ssh readiness timed out (last: %v)", lastErr)
|
||||||
|
}
|
||||||
|
key, err := h.handshake(addr)
|
||||||
|
if err == nil {
|
||||||
|
return key, nil
|
||||||
|
}
|
||||||
|
lastErr = err
|
||||||
|
time.Sleep(500 * time.Millisecond)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// handshake performs one full SSH exchange and captures the server host
|
||||||
|
// key via the callback so the harness can write a known_hosts entry.
|
||||||
|
// A fresh known_hosts file trusts the first key it sees (trust-on-first-
|
||||||
|
// use); the host-key-mismatch integration test proves that a known_hosts
|
||||||
|
// entry carrying a *different* key is rejected before any command runs.
|
||||||
|
func (h *Harness) handshake(addr string) (ssh.PublicKey, error) {
|
||||||
|
keyBytes, err := os.ReadFile(testKeyPath())
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
signer, err := ssh.ParsePrivateKey(keyBytes)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
var got ssh.PublicKey
|
||||||
|
config := &ssh.ClientConfig{
|
||||||
|
User: h.user,
|
||||||
|
Auth: []ssh.AuthMethod{ssh.PublicKeys(signer)},
|
||||||
|
HostKeyCallback: func(_ string, _ net.Addr, key ssh.PublicKey) error {
|
||||||
|
got = key
|
||||||
|
return nil
|
||||||
|
},
|
||||||
|
Timeout: 5 * time.Second,
|
||||||
|
}
|
||||||
|
client, err := ssh.Dial("tcp", addr, config)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer client.Close() //nolint:errcheck
|
||||||
|
session, err := client.NewSession()
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer session.Close() //nolint:errcheck
|
||||||
|
if _, err := session.Output("true"); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if got == nil {
|
||||||
|
return nil, errors.New("no host key returned by handshake")
|
||||||
|
}
|
||||||
|
return got, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// waitForPort polls a TCP endpoint until it accepts a connection. It is
|
||||||
|
// a plain network probe; SSH readiness is separately verified.
|
||||||
|
func waitForPort(ctx context.Context, addr string, timeout time.Duration) error {
|
||||||
|
deadline := time.Now().Add(timeout)
|
||||||
|
for {
|
||||||
|
if ctx.Err() != nil {
|
||||||
|
return ctx.Err()
|
||||||
|
}
|
||||||
|
if time.Now().After(deadline) {
|
||||||
|
return fmt.Errorf("port %s not open within %s", addr, timeout)
|
||||||
|
}
|
||||||
|
conn, err := net.DialTimeout("tcp", addr, 2*time.Second)
|
||||||
|
if err == nil {
|
||||||
|
_ = conn.Close()
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
time.Sleep(500 * time.Millisecond)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// writeKnownHosts writes a known_hosts entry for the harness address
|
||||||
|
// inside an existing temp directory, using the captured server host key.
|
||||||
|
func writeKnownHosts(dir, addr string, key ssh.PublicKey) (string, error) {
|
||||||
|
path := filepath.Join(dir, "known_hosts")
|
||||||
|
line := knownhosts.Line([]string{addr}, key)
|
||||||
|
if err := os.WriteFile(path, []byte(line+"\n"), 0o600); err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
return path, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// testKeyPath returns the shared test private key bundled with the
|
||||||
|
// package. The fixtures bake the matching public key into authorized_keys.
|
||||||
|
//
|
||||||
|
// This keypair is strictly test-only: it grants root SSH access only to
|
||||||
|
// the disposable fixture containers that bake its public key. It must
|
||||||
|
// never be used for real hosts, copied into production images, or
|
||||||
|
// treated as a credential anywhere outside the harness.
|
||||||
|
func testKeyPath() string {
|
||||||
|
return filepath.Join("testdata", "keys", "rsmon_test_ed25519")
|
||||||
|
}
|
||||||
|
|
||||||
|
// validateName rejects harness names that could inject shell or docker
|
||||||
|
// metacharacters into container/network names.
|
||||||
|
func validateName(name string) error {
|
||||||
|
if name == "" || len(name) > 64 {
|
||||||
|
return errors.New("harness name must be 1-64 characters")
|
||||||
|
}
|
||||||
|
for _, r := range name {
|
||||||
|
lower := r >= 'a' && r <= 'z'
|
||||||
|
digit := r >= '0' && r <= '9'
|
||||||
|
if !lower && !digit && r != '-' {
|
||||||
|
return fmt.Errorf("harness name %q must be lowercase alphanumeric and hyphens", name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func sanitize(name string) string {
|
||||||
|
var b strings.Builder
|
||||||
|
for _, r := range name {
|
||||||
|
if r >= 'a' && r <= 'z' || r >= '0' && r <= '9' {
|
||||||
|
b.WriteRune(r)
|
||||||
|
} else {
|
||||||
|
b.WriteByte('-')
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return b.String()
|
||||||
|
}
|
||||||
417
internal/installer/harness/harness_test.go
Обычный файл
417
internal/installer/harness/harness_test.go
Обычный файл
@@ -0,0 +1,417 @@
|
|||||||
|
package harness
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"net"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"golang.org/x/crypto/ssh"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestEnabled(t *testing.T) {
|
||||||
|
// os.Unsetenv would leak into later tests in the same process;
|
||||||
|
// t.Setenv restores the original value after this test.
|
||||||
|
t.Setenv("RSMON_TEST_DOCKER", "")
|
||||||
|
if Enabled() {
|
||||||
|
t.Fatal("Enabled() true without RSMON_TEST_DOCKER")
|
||||||
|
}
|
||||||
|
for _, v := range []string{"1", "true", "TRUE", "yes", "on"} {
|
||||||
|
t.Setenv("RSMON_TEST_DOCKER", v)
|
||||||
|
if !Enabled() {
|
||||||
|
t.Fatalf("Enabled() false for RSMON_TEST_DOCKER=%q", v)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, v := range []string{"0", "false", "no", "off", " "} {
|
||||||
|
t.Setenv("RSMON_TEST_DOCKER", v)
|
||||||
|
if Enabled() {
|
||||||
|
t.Fatalf("Enabled() true for RSMON_TEST_DOCKER=%q", v)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFixtureDefaults(t *testing.T) {
|
||||||
|
fixtures := Fixtures()
|
||||||
|
if len(fixtures) != 3 {
|
||||||
|
t.Fatalf("Fixtures() = %d, want 3", len(fixtures))
|
||||||
|
}
|
||||||
|
byName := map[string]Fixture{}
|
||||||
|
for _, f := range fixtures {
|
||||||
|
byName[f.Name] = f
|
||||||
|
}
|
||||||
|
if f := byName["alpine"]; f.ImageRef() != "reg.rsxx.ru/library/alpine:3" || f.Distro != "alpine" {
|
||||||
|
t.Fatalf("alpine fixture wrong: %+v", f)
|
||||||
|
}
|
||||||
|
if f := byName["ubuntu"]; f.ImageRef() != "ubuntu:24.04" || f.Distro != "ubuntu" {
|
||||||
|
t.Fatalf("ubuntu fixture wrong: %+v", f)
|
||||||
|
}
|
||||||
|
if f := byName["arch"]; f.ImageRef() != "archlinux:latest" || f.Distro != "arch" {
|
||||||
|
t.Fatalf("arch fixture wrong: %+v", f)
|
||||||
|
}
|
||||||
|
if f := byName["alpine"]; f.UserOrDefault() != "root" {
|
||||||
|
t.Fatalf("default user = %q, want root", f.UserOrDefault())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFixtureImageEnvOverride(t *testing.T) {
|
||||||
|
f := Fixture{Name: "ubuntu"}
|
||||||
|
t.Setenv("RSMON_TEST_IMAGE_UBUNTU", "reg.rsxx.ru/library/ubuntu:24.04")
|
||||||
|
if got := f.ImageRef(); got != "reg.rsxx.ru/library/ubuntu:24.04" {
|
||||||
|
t.Fatalf("env override not applied: %q", got)
|
||||||
|
}
|
||||||
|
t.Setenv("RSMON_TEST_IMAGE_UBUNTU", "")
|
||||||
|
if got := f.ImageRef(); got != "ubuntu:24.04" {
|
||||||
|
t.Fatalf("default image changed after env cleared: %q", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
override := Fixture{Name: "arch", Image: "archlinux:2026.01.01"}
|
||||||
|
if got := override.ImageRef(); got != "archlinux:2026.01.01" {
|
||||||
|
t.Fatalf("fixture image override not applied: %q", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFixturesHaveDockerfiles(t *testing.T) {
|
||||||
|
for _, f := range Fixtures() {
|
||||||
|
path := f.DockerfilePath()
|
||||||
|
if _, err := os.Stat(path); err != nil {
|
||||||
|
t.Fatalf("fixture %s missing Dockerfile at %s: %v", f.Name, path, err)
|
||||||
|
}
|
||||||
|
data, err := os.ReadFile(path)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !strings.Contains(string(data), "FROM ") {
|
||||||
|
t.Fatalf("fixture %s Dockerfile has no FROM", f.Name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNewValidatesName(t *testing.T) {
|
||||||
|
for _, bad := range []string{"", "with space", "UPPER", "semi;colon", "back`tick", strings.Repeat("a", 65)} {
|
||||||
|
if _, err := New(bad, Fixtures()[0]); err == nil {
|
||||||
|
t.Fatalf("New(%q) succeeded", bad)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
h, err := New("fixture-alpine", Fixtures()[0])
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if h.ContainerName() == "" || h.NetworkName() == "" {
|
||||||
|
t.Fatalf("empty container/network names: %+v", h)
|
||||||
|
}
|
||||||
|
if h.Addr() != "" {
|
||||||
|
t.Fatalf("Addr() = %q before Start, want empty", h.Addr())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestStartDisabled(t *testing.T) {
|
||||||
|
t.Setenv("RSMON_TEST_DOCKER", "")
|
||||||
|
h, err := New("unit-start", Fixtures()[0])
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := h.Start(context.Background()); err == nil {
|
||||||
|
t.Fatal("Start succeeded with integration disabled")
|
||||||
|
} else if !strings.Contains(err.Error(), "RSMON_TEST_DOCKER") {
|
||||||
|
t.Fatalf("disabled Start error = %v, want opt-in hint", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDialBeforeStart(t *testing.T) {
|
||||||
|
h, err := New("unit-dial", Fixtures()[0])
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := h.Dial(); err == nil {
|
||||||
|
t.Fatal("Dial succeeded before Start")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParsePublishedPort(t *testing.T) {
|
||||||
|
for _, tc := range []struct {
|
||||||
|
in string
|
||||||
|
want int
|
||||||
|
}{
|
||||||
|
{in: "127.0.0.1:49153\n", want: 49153},
|
||||||
|
{in: "0.0.0.0:2222", want: 2222},
|
||||||
|
{in: "127.0.0.1:0\n", want: 0},
|
||||||
|
{in: "::1:32768\n", want: 32768},
|
||||||
|
{in: "127.0.0.1:notaport\n", want: 0},
|
||||||
|
{in: "", want: 0},
|
||||||
|
} {
|
||||||
|
got, err := parsePublishedPort(tc.in)
|
||||||
|
if tc.want == 0 {
|
||||||
|
if err == nil {
|
||||||
|
t.Fatalf("parsePublishedPort(%q) succeeded with %d", tc.in, got)
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if err != nil || got != tc.want {
|
||||||
|
t.Fatalf("parsePublishedPort(%q) = %d, %v; want %d", tc.in, got, err, tc.want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestWaitForPort exercises the readiness probe against a real local
|
||||||
|
// listener so the polling loop is covered without Docker.
|
||||||
|
func TestWaitForPort(t *testing.T) {
|
||||||
|
ln, err := net.Listen("tcp", "127.0.0.1:0")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer ln.Close() //nolint:errcheck
|
||||||
|
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
if err := waitForPort(ctx, ln.Addr().String(), 2*time.Second); err != nil {
|
||||||
|
t.Fatalf("waitForPort on open listener: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// A port that never opens must time out.
|
||||||
|
closed, err := net.Listen("tcp", "127.0.0.1:0")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
addr := closed.Addr().String()
|
||||||
|
closed.Close() //nolint:errcheck
|
||||||
|
|
||||||
|
short, shortCancel := context.WithTimeout(context.Background(), 3*time.Second)
|
||||||
|
defer shortCancel()
|
||||||
|
if err := waitForPort(short, addr, 1500*time.Millisecond); err == nil {
|
||||||
|
t.Fatal("waitForPort on closed port succeeded")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestWriteKnownHostsRoundTrip(t *testing.T) {
|
||||||
|
// Parsing a known_hosts entry requires a real key; use the bundled
|
||||||
|
// public key so the format is exercised.
|
||||||
|
raw, err := os.ReadFile(filepath.Join("testdata", "keys", "rsmon_test_ed25519.pub"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
key, _, _, _, err := ssh.ParseAuthorizedKey(raw)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
path, err := writeKnownHosts(t.TempDir(), "127.0.0.1:49153", key)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
data, err := os.ReadFile(path)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !strings.Contains(string(data), "[127.0.0.1]:49153") {
|
||||||
|
t.Fatalf("known_hosts entry %q missing bracketed address", data)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestValidateName(t *testing.T) {
|
||||||
|
for _, ok := range []string{"alpine", "fixture-1", "a", strings.Repeat("x", 64)} {
|
||||||
|
if err := validateName(ok); err != nil {
|
||||||
|
t.Fatalf("validateName(%q): %v", ok, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, bad := range []string{"", "A", "a b", "a/b", strings.Repeat("x", 65)} {
|
||||||
|
if err := validateName(bad); err == nil {
|
||||||
|
t.Fatalf("validateName(%q) succeeded", bad)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSetDockerBin(t *testing.T) {
|
||||||
|
SetDockerBin("docker")
|
||||||
|
SetDockerBin("")
|
||||||
|
if dockerBin != "docker" {
|
||||||
|
t.Fatalf("dockerBin = %q after reset", dockerBin)
|
||||||
|
}
|
||||||
|
SetDockerBin("/stub/docker")
|
||||||
|
if dockerBin != "/stub/docker" {
|
||||||
|
t.Fatalf("dockerBin = %q after set", dockerBin)
|
||||||
|
}
|
||||||
|
SetDockerBin("")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDockerDNSOverride(t *testing.T) {
|
||||||
|
t.Setenv("RSMON_TEST_DOCKER_DNS", "")
|
||||||
|
if got := dockerDNS(); len(got) != 0 {
|
||||||
|
t.Fatalf("dockerDNS() with empty env = %v, want none", got)
|
||||||
|
}
|
||||||
|
t.Setenv("RSMON_TEST_DOCKER_DNS", "8.8.8.8, 1.1.1.1")
|
||||||
|
if got := dockerDNS(); len(got) != 4 || got[0] != "--dns" || got[1] != "8.8.8.8" || got[3] != "1.1.1.1" {
|
||||||
|
t.Fatalf("dockerDNS() = %v", got)
|
||||||
|
}
|
||||||
|
t.Setenv("RSMON_TEST_DOCKER_DNS", " ,,")
|
||||||
|
if got := dockerDNS(); len(got) != 0 {
|
||||||
|
t.Fatalf("dockerDNS() with blank entries = %v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// writeStubDocker installs a fake docker binary that records its argv to
|
||||||
|
// logPath and returns the recorded path. The stub succeeds for build,
|
||||||
|
// network, run, and teardown calls; `port` fails so Start fails after
|
||||||
|
// the container and network exist. Setting STUB_DOCKER_FAIL_BUILD=1 makes
|
||||||
|
// the build step fail instead. Real Docker is never touched.
|
||||||
|
func writeStubDocker(t *testing.T, logPath string) string {
|
||||||
|
t.Helper()
|
||||||
|
dir := t.TempDir()
|
||||||
|
stub := filepath.Join(dir, "docker")
|
||||||
|
script := `#!/bin/sh
|
||||||
|
echo "$*" >> "$STUB_DOCKER_LOG"
|
||||||
|
case "$1" in
|
||||||
|
build)
|
||||||
|
if [ "${STUB_DOCKER_FAIL_BUILD:-0}" = "1" ]; then
|
||||||
|
echo "stub build failure" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
echo "sha256:stub-image-id"
|
||||||
|
;;
|
||||||
|
network) echo "stub-network-id" ;;
|
||||||
|
run) echo "stub-container-id" ;;
|
||||||
|
port) echo "stub port failure" >&2; exit 1 ;;
|
||||||
|
rm|-r|image|ps) exit 0 ;;
|
||||||
|
*) exit 0 ;;
|
||||||
|
esac
|
||||||
|
`
|
||||||
|
if err := os.WriteFile(stub, []byte(script), 0o700); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Setenv("STUB_DOCKER_LOG", logPath)
|
||||||
|
SetDockerBin(stub)
|
||||||
|
t.Cleanup(func() { SetDockerBin("") })
|
||||||
|
return stub
|
||||||
|
}
|
||||||
|
|
||||||
|
func readDockerCalls(t *testing.T, logPath string) string {
|
||||||
|
t.Helper()
|
||||||
|
calls, err := os.ReadFile(logPath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return string(calls)
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestStartFailureCleansUpResources drives Start to failure after the
|
||||||
|
// container and network were created (the stub `docker port` fails) and
|
||||||
|
// asserts the failure path removed every resource: container, network,
|
||||||
|
// fixture image, and no temp known_hosts dir leaked.
|
||||||
|
func TestStartFailureCleansUpResources(t *testing.T) {
|
||||||
|
t.Setenv("RSMON_TEST_DOCKER", "1")
|
||||||
|
logPath := filepath.Join(t.TempDir(), "docker-calls.log")
|
||||||
|
writeStubDocker(t, logPath)
|
||||||
|
|
||||||
|
h, err := New("fail-cleanup", Fixtures()[0])
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := h.Start(context.Background()); err == nil {
|
||||||
|
t.Fatal("Start succeeded despite stub docker port failure")
|
||||||
|
}
|
||||||
|
calls := readDockerCalls(t, logPath)
|
||||||
|
for _, want := range []string{"build", "network create", "run -d", "rm -f", "network rm", "image rm"} {
|
||||||
|
if !strings.Contains(calls, want) {
|
||||||
|
t.Fatalf("failed-start cleanup missing docker call %q; calls:\n%s", want, calls)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if h.KnownHostsPath() != "" {
|
||||||
|
t.Fatalf("known_hosts path set after failed start: %q", h.KnownHostsPath())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestStartBuildFailureIsSafe drives Start to fail at the very first
|
||||||
|
// step (build) and asserts teardown stays idempotent and harmless: no
|
||||||
|
// container or network was ever created, and the failure path leaves
|
||||||
|
// nothing behind.
|
||||||
|
func TestStartBuildFailureIsSafe(t *testing.T) {
|
||||||
|
t.Setenv("RSMON_TEST_DOCKER", "1")
|
||||||
|
t.Setenv("STUB_DOCKER_FAIL_BUILD", "1")
|
||||||
|
logPath := filepath.Join(t.TempDir(), "docker-calls.log")
|
||||||
|
writeStubDocker(t, logPath)
|
||||||
|
|
||||||
|
h, err := New("fail-build", Fixtures()[0])
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := h.Start(context.Background()); err == nil {
|
||||||
|
t.Fatal("Start succeeded despite stub docker build failure")
|
||||||
|
}
|
||||||
|
if h.ContainerID() != "" {
|
||||||
|
t.Fatalf("container id set after build failure: %q", h.ContainerID())
|
||||||
|
}
|
||||||
|
if h.KnownHostsPath() != "" {
|
||||||
|
t.Fatalf("known_hosts path set after build failure: %q", h.KnownHostsPath())
|
||||||
|
}
|
||||||
|
if err := h.Stop(); err != nil {
|
||||||
|
t.Fatalf("Stop after failed build: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestStopRemovesKnownHostsDir verifies Stop deletes the temp known_hosts
|
||||||
|
// directory even when no container was ever started (the failed-start
|
||||||
|
// path registers the dir before the final write).
|
||||||
|
func TestStopRemovesKnownHostsDir(t *testing.T) {
|
||||||
|
logPath := filepath.Join(t.TempDir(), "docker-calls.log")
|
||||||
|
writeStubDocker(t, logPath)
|
||||||
|
|
||||||
|
h, err := New("stop-dir", Fixtures()[0])
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
dir := t.TempDir()
|
||||||
|
h.knownHosts = filepath.Join(dir, "known_hosts")
|
||||||
|
if err := os.WriteFile(h.knownHosts, []byte("placeholder"), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := h.Stop(); err != nil {
|
||||||
|
t.Fatalf("Stop: %v", err)
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(dir); !os.IsNotExist(err) {
|
||||||
|
t.Fatalf("known_hosts temp dir still exists after Stop: %v", err)
|
||||||
|
}
|
||||||
|
if h.KnownHostsPath() != "" {
|
||||||
|
t.Fatalf("KnownHostsPath = %q after Stop, want empty", h.KnownHostsPath())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestStopIdempotent verifies repeated Stop calls do not error.
|
||||||
|
func TestStopIdempotent(t *testing.T) {
|
||||||
|
logPath := filepath.Join(t.TempDir(), "docker-calls.log")
|
||||||
|
writeStubDocker(t, logPath)
|
||||||
|
|
||||||
|
h, err := New("stop-again", Fixtures()[0])
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
h.container = "rsmon-worker-test-none"
|
||||||
|
h.network = "rsmon-worker-test-none"
|
||||||
|
h.imageTag = "rsmon-worker-test/none:local"
|
||||||
|
h.knownHosts = filepath.Join(t.TempDir(), "known_hosts")
|
||||||
|
if err := h.Stop(); err != nil {
|
||||||
|
t.Fatalf("first Stop: %v", err)
|
||||||
|
}
|
||||||
|
if err := h.Stop(); err != nil {
|
||||||
|
t.Fatalf("second Stop: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestImageTagUnique verifies every harness instance gets its own image
|
||||||
|
// tag so teardown can never delete another instance's image.
|
||||||
|
func TestImageTagUnique(t *testing.T) {
|
||||||
|
a, err := New("img-a", Fixtures()[0])
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
b, err := New("img-b", Fixtures()[0])
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if a.ImageTag() == b.ImageTag() {
|
||||||
|
t.Fatalf("image tags collide: %q", a.ImageTag())
|
||||||
|
}
|
||||||
|
if !strings.HasPrefix(a.ImageTag(), "rsmon-worker-test/alpine-") {
|
||||||
|
t.Fatalf("unexpected image tag: %q", a.ImageTag())
|
||||||
|
}
|
||||||
|
}
|
||||||
393
internal/installer/harness/integration_activation_test.go
Обычный файл
393
internal/installer/harness/integration_activation_test.go
Обычный файл
@@ -0,0 +1,393 @@
|
|||||||
|
package harness
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"golang.org/x/crypto/ssh"
|
||||||
|
|
||||||
|
"rocketgit.ru/rsmon/worker/internal/installer"
|
||||||
|
"rocketgit.ru/rsmon/worker/internal/sshinstall"
|
||||||
|
)
|
||||||
|
|
||||||
|
// testWorkerToken is the operator-supplied worker token used by the
|
||||||
|
// activation E2E tests. The fixture has no reachable control plane, but
|
||||||
|
// the worker still binds its webapp and serves /healthz, which is what
|
||||||
|
// activation verifies.
|
||||||
|
const testWorkerToken = "e2e-activation-test-token"
|
||||||
|
|
||||||
|
// activationOptions builds the work-package-4 source-install options for
|
||||||
|
// a started fixture: full activation with a fixed test token, honoring
|
||||||
|
// the RSMON_TEST_SOURCE_REPO/BRANCH overrides the staging tests use.
|
||||||
|
func activationOptions(h *Harness, f Fixture) installer.SourceInstallOptions {
|
||||||
|
opts := installer.SourceInstallOptions{
|
||||||
|
SSHOptions: installer.SSHOptions{
|
||||||
|
Host: "127.0.0.1",
|
||||||
|
Port: h.Port(),
|
||||||
|
User: f.UserOrDefault(),
|
||||||
|
IdentityFile: testKeyPath(),
|
||||||
|
KnownHostsFile: h.KnownHostsPath(),
|
||||||
|
},
|
||||||
|
Activation: installer.ActivationOptions{
|
||||||
|
Activate: true,
|
||||||
|
URL: "https://rsmon.ru",
|
||||||
|
Token: testWorkerToken,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
if repo := strings.TrimSpace(os.Getenv("RSMON_TEST_SOURCE_REPO")); repo != "" {
|
||||||
|
opts.Repo = repo
|
||||||
|
}
|
||||||
|
if branch := strings.TrimSpace(os.Getenv("RSMON_TEST_SOURCE_BRANCH")); branch != "" {
|
||||||
|
opts.Branch = branch
|
||||||
|
}
|
||||||
|
return opts
|
||||||
|
}
|
||||||
|
|
||||||
|
// fixtureByName returns the fixture entry for a fixture name.
|
||||||
|
func fixtureByName(t *testing.T, name string) Fixture {
|
||||||
|
t.Helper()
|
||||||
|
for _, f := range Fixtures() {
|
||||||
|
if f.Name == name {
|
||||||
|
return f
|
||||||
|
}
|
||||||
|
}
|
||||||
|
t.Fatalf("no fixture named %q", name)
|
||||||
|
return Fixture{}
|
||||||
|
}
|
||||||
|
|
||||||
|
// startActivatedFixture starts a fixture container and runs a full
|
||||||
|
// source install with activation against it, returning the harness, the
|
||||||
|
// options (so a test can rerun with modifications), the result, and a
|
||||||
|
// live SSH client. The caller owns client.Close and h.Stop (registered as
|
||||||
|
// a test cleanup).
|
||||||
|
func startActivatedFixture(t *testing.T, name string) (*Harness, installer.SourceInstallOptions, *installer.SourceInstallResult, *ssh.Client) {
|
||||||
|
t.Helper()
|
||||||
|
f := fixtureByName(t, name)
|
||||||
|
h, err := New("activate-"+name, f)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute)
|
||||||
|
defer cancel()
|
||||||
|
if err := h.Start(ctx); err != nil {
|
||||||
|
t.Fatalf("start %s fixture: %v", name, err)
|
||||||
|
}
|
||||||
|
t.Cleanup(func() {
|
||||||
|
if err := h.Stop(); err != nil {
|
||||||
|
t.Errorf("stop %s fixture: %v", name, err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
opts := activationOptions(h, f)
|
||||||
|
res, err := installer.SourceInstall(opts)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("activated source install on %s: %v", name, err)
|
||||||
|
}
|
||||||
|
client, err := h.Dial()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("dial %s after install: %v", name, err)
|
||||||
|
}
|
||||||
|
return h, opts, res, client
|
||||||
|
}
|
||||||
|
|
||||||
|
// countWorkerProcesses returns the number of running processes whose
|
||||||
|
// command line is exactly the installed worker binary. The anchored -f
|
||||||
|
// pattern matches both busybox and procps pgrep (busybox's -x matches the
|
||||||
|
// full argv, so -x cannot be used portably).
|
||||||
|
func countWorkerProcesses(t *testing.T, client *ssh.Client, binary string) string {
|
||||||
|
t.Helper()
|
||||||
|
out, err := RunCommand(client, "pgrep -f '^"+binary+"$' | wc -l")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("count worker processes: %v", err)
|
||||||
|
}
|
||||||
|
return strings.TrimSpace(string(out))
|
||||||
|
}
|
||||||
|
|
||||||
|
// assertActivatedService verifies the full work-package-4 remote state:
|
||||||
|
// the installed layout and permissions, the native service definition for
|
||||||
|
// the detected init, exactly one running worker process, the pid file,
|
||||||
|
// and a live /healthz (via the installed binary's own liveness
|
||||||
|
// subcommand).
|
||||||
|
func assertActivatedService(t *testing.T, client *ssh.Client, name string, res *installer.SourceInstallResult) {
|
||||||
|
t.Helper()
|
||||||
|
a := res.Activation
|
||||||
|
if a == nil {
|
||||||
|
t.Fatalf("%s: activation result missing", name)
|
||||||
|
}
|
||||||
|
|
||||||
|
out, err := RunCommand(client, "test -x "+shellQuote(a.Binary)+" && echo BIN_OK")
|
||||||
|
if err != nil || !strings.Contains(string(out), "BIN_OK") {
|
||||||
|
t.Fatalf("%s: installed binary not present/executable at %s: %q, %v", name, a.Binary, out, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
envMode, err := RunCommand(client, "stat -c '%a' "+shellQuote(a.EnvFile))
|
||||||
|
if err != nil || strings.TrimSpace(string(envMode)) != "600" {
|
||||||
|
t.Fatalf("%s: env file mode = %q, want 600 (%v)", name, envMode, err)
|
||||||
|
}
|
||||||
|
cfgMode, err := RunCommand(client, "stat -c '%a' "+shellQuote(a.ConfigDir))
|
||||||
|
if err != nil || strings.TrimSpace(string(cfgMode)) != "750" {
|
||||||
|
t.Fatalf("%s: config dir mode = %q, want 750 (%v)", name, cfgMode, err)
|
||||||
|
}
|
||||||
|
if _, err := RunCommand(client, "test -d "+shellQuote(a.DataDir)+"/webapp && echo DATA_OK"); err != nil {
|
||||||
|
t.Fatalf("%s: data dir not created at %s: %v", name, a.DataDir, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// The fixtures run no init system (sshd is PID 1), so activation
|
||||||
|
// must use the embedded supervisor; the native service definition is
|
||||||
|
// still installed for real hosts.
|
||||||
|
if a.Supervisor != "none" {
|
||||||
|
t.Fatalf("%s: supervisor = %q, want none in the fixture", name, a.Supervisor)
|
||||||
|
}
|
||||||
|
unitMode := "644"
|
||||||
|
if res.Detection.InitSystem == sshinstall.InitOpenRC {
|
||||||
|
unitMode = "755"
|
||||||
|
}
|
||||||
|
if a.UnitFile == "" {
|
||||||
|
t.Fatalf("%s: no service definition installed", name)
|
||||||
|
}
|
||||||
|
if _, err := RunCommand(client, "test -f "+shellQuote(a.UnitFile)+" && echo UNIT_OK"); err != nil {
|
||||||
|
t.Fatalf("%s: unit file missing at %s: %v", name, a.UnitFile, err)
|
||||||
|
}
|
||||||
|
gotUnitMode, err := RunCommand(client, "stat -c '%a' "+shellQuote(a.UnitFile))
|
||||||
|
if err != nil || strings.TrimSpace(string(gotUnitMode)) != unitMode {
|
||||||
|
t.Fatalf("%s: unit mode = %q, want %s (%v)", name, gotUnitMode, unitMode, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Exactly one worker process and a live /healthz.
|
||||||
|
if got := countWorkerProcesses(t, client, a.Binary); got != "1" {
|
||||||
|
t.Fatalf("%s: worker process count = %q, want 1", name, got)
|
||||||
|
}
|
||||||
|
if out, err := RunCommand(client, shellQuote(a.Binary)+" liveness"); err != nil || !strings.Contains(string(out), "ok") {
|
||||||
|
t.Fatalf("%s: installed worker /healthz check failed: %q, %v", name, out, err)
|
||||||
|
}
|
||||||
|
pidOut, err := RunCommand(client, "cat "+shellQuote(a.DataDir)+"/worker.pid")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("%s: pid file missing: %v", name, err)
|
||||||
|
}
|
||||||
|
if _, err := RunCommand(client, "kill -0 "+strings.TrimSpace(string(pidOut))+" && echo PID_OK"); err != nil {
|
||||||
|
t.Fatalf("%s: pid %s not alive: %v", name, pidOut, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// assertNoActivationLeaks verifies a rerun left no backup dir, no lock,
|
||||||
|
// no /tmp upload leftovers, and no extra worker processes.
|
||||||
|
func assertNoActivationLeaks(t *testing.T, client *ssh.Client, name string, dataDir string) {
|
||||||
|
t.Helper()
|
||||||
|
out, err := RunCommand(client,
|
||||||
|
"leak=0; [ -e "+shellQuote(dataDir)+"/.rsmon-backup ] && leak=1; "+
|
||||||
|
"[ -d "+shellQuote(dataDir)+"/.rsmon-activate.lock ] && leak=1; "+
|
||||||
|
"ls /tmp 2>/dev/null | grep -q '^rsmon-worker-act\\.' && leak=1; "+
|
||||||
|
"[ \"$leak\" -eq 1 ] && echo LEAK || echo CLEAN")
|
||||||
|
if err != nil || strings.TrimSpace(string(out)) != "CLEAN" {
|
||||||
|
t.Fatalf("%s: activation leftovers after rerun: %q, %v", name, out, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// installedBinarySHA captures the SHA-256 of the installed worker binary
|
||||||
|
// so failure tests can prove the prior binary is preserved.
|
||||||
|
func installedBinarySHA(t *testing.T, client *ssh.Client, a *installer.ActivationResult) []byte {
|
||||||
|
t.Helper()
|
||||||
|
out, err := RunCommand(client, "sha256sum "+shellQuote(a.Binary))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return bytes.TrimSpace(out)
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestSourceInstallActivationFixtures is the work-package-4 success and
|
||||||
|
// idempotency acceptance test: on every fixture a full source install
|
||||||
|
// ends with an atomically installed binary, a mode-0600 env file, the
|
||||||
|
// data dir, a service definition for the detected init, and a running
|
||||||
|
// verified worker. A rerun succeeds, keeps a single worker process,
|
||||||
|
// rewrites the env deterministically, and leaks no temp or backup files.
|
||||||
|
//
|
||||||
|
// Opt-in: set RSMON_TEST_DOCKER=1 (see make test-ssh).
|
||||||
|
func TestSourceInstallActivationFixtures(t *testing.T) {
|
||||||
|
SkipUnlessEnabled(t)
|
||||||
|
|
||||||
|
for _, f := range Fixtures() {
|
||||||
|
f := f
|
||||||
|
t.Run(f.Name, func(t *testing.T) {
|
||||||
|
_, opts, res, client := startActivatedFixture(t, f.Name)
|
||||||
|
defer client.Close() //nolint:errcheck
|
||||||
|
assertActivatedService(t, client, f.Name, res)
|
||||||
|
|
||||||
|
// Rerun: idempotent, single service, no leaks.
|
||||||
|
res2, err := installer.SourceInstall(opts)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("%s: activated rerun: %v", f.Name, err)
|
||||||
|
}
|
||||||
|
if res2.Activation == nil || res2.ResolvedCommit != res.ResolvedCommit {
|
||||||
|
t.Fatalf("%s: rerun lost activation/commit: %+v", f.Name, res2)
|
||||||
|
}
|
||||||
|
assertActivatedService(t, client, f.Name, res2)
|
||||||
|
assertNoActivationLeaks(t, client, f.Name, res2.Activation.DataDir)
|
||||||
|
|
||||||
|
envAfter, err := RunCommand(client, "cat "+shellQuote(res2.Activation.EnvFile))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("%s: read env after rerun: %v", f.Name, err)
|
||||||
|
}
|
||||||
|
if !strings.Contains(string(envAfter), "RSMON_TOKEN="+testWorkerToken) {
|
||||||
|
t.Fatalf("%s: env lost the worker token after rerun: %q", f.Name, envAfter)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestSourceInstallActivationFailureRollback is the work-package-4
|
||||||
|
// rollback acceptance test. After a successful activated install on the
|
||||||
|
// Alpine fixture, each rerun is forced to fail at a different stage -
|
||||||
|
// build (dirty checkout), activation (atomic swap sabotaged), start (the
|
||||||
|
// worker process dies at boot), and health (/healthz unreachable) - and
|
||||||
|
// every failure must leave the previous working install running with the
|
||||||
|
// same binary, the same single process, and /healthz answering.
|
||||||
|
//
|
||||||
|
// Opt-in: set RSMON_TEST_DOCKER=1 (see make test-ssh).
|
||||||
|
func TestSourceInstallActivationFailureRollback(t *testing.T) {
|
||||||
|
SkipUnlessEnabled(t)
|
||||||
|
|
||||||
|
const fixture = "alpine"
|
||||||
|
_, opts, res, client := startActivatedFixture(t, fixture)
|
||||||
|
defer client.Close() //nolint:errcheck
|
||||||
|
a := res.Activation
|
||||||
|
assertActivatedService(t, client, fixture, res)
|
||||||
|
|
||||||
|
// 1. Build failure preserves the prior install.
|
||||||
|
// A dirty tracked tree makes the rerun's checkout refuse before
|
||||||
|
// the build, so neither staging nor the service changes. Appending
|
||||||
|
// a marker is deterministic regardless of the cloned history (a
|
||||||
|
// `git checkout master~1 -- <file>` would depend on whether that
|
||||||
|
// commit differs from the resolved one).
|
||||||
|
if _, err := RunCommand(client, "printf '\\n# rsmon-worker dirty-tree marker\\n' >> "+shellQuote(res.Plan.BuildDir)+"/Makefile"); err != nil {
|
||||||
|
t.Fatalf("dirty the working tree: %v", err)
|
||||||
|
}
|
||||||
|
before := installedBinarySHA(t, client, a)
|
||||||
|
_, err := installer.SourceInstall(opts)
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "check out branch") {
|
||||||
|
t.Fatalf("rerun on dirty tree err = %v, want checkout failure", err)
|
||||||
|
}
|
||||||
|
assertActivatedService(t, client, fixture, res)
|
||||||
|
if !bytes.Equal(before, installedBinarySHA(t, client, a)) {
|
||||||
|
t.Fatalf("installed binary changed after build-failure rerun")
|
||||||
|
}
|
||||||
|
// Restore the tree so later reruns can check out again.
|
||||||
|
if _, err := RunCommand(client, "git -C "+shellQuote(res.Plan.BuildDir)+" checkout -q HEAD -- Makefile"); err != nil {
|
||||||
|
t.Fatalf("restore working tree: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// 2. Activation failure preserves the prior install.
|
||||||
|
// Pre-creating the atomic-swap temp path as a directory makes the
|
||||||
|
// binary install step fail mid-activation; the rollback trap must
|
||||||
|
// restore the previous binary/env and restart the prior service.
|
||||||
|
if _, err := RunCommand(client, "mkdir -p /usr/local/bin/rsmon-worker.new"); err != nil {
|
||||||
|
t.Fatalf("sabotage the atomic binary swap: %v", err)
|
||||||
|
}
|
||||||
|
_, err = installer.SourceInstall(opts)
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "activate worker service") {
|
||||||
|
t.Fatalf("activation-failure rerun err = %v, want activate failure", err)
|
||||||
|
}
|
||||||
|
assertActivatedService(t, client, fixture, res)
|
||||||
|
if !bytes.Equal(before, installedBinarySHA(t, client, a)) {
|
||||||
|
t.Fatalf("installed binary changed after activation-failure rerun")
|
||||||
|
}
|
||||||
|
assertNoActivationLeaks(t, client, fixture, a.DataDir)
|
||||||
|
if _, err := RunCommand(client, "rm -rf /usr/local/bin/rsmon-worker.new"); err != nil {
|
||||||
|
t.Fatalf("clean the swap sabotage: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// 3. Start failure preserves the prior install.
|
||||||
|
// WORKER_CLUSTER_ENABLED=true without WORKER_LOGIN/PASSWORD makes
|
||||||
|
// the new worker exit at boot (cluster init is fatal), so the
|
||||||
|
// activation supervisor detects the process died and rolls back.
|
||||||
|
envPath := filepath.Join(t.TempDir(), "worker.env")
|
||||||
|
startEnv := "RSMON_URL=https://rsmon.ru\nRSMON_TOKEN=" + testWorkerToken + "\nWORKER_CLUSTER_ENABLED=true\n"
|
||||||
|
if err := os.WriteFile(envPath, []byte(startEnv), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
startOpts := opts
|
||||||
|
startOpts.Activation.EnvFile = envPath
|
||||||
|
_, err = installer.SourceInstall(startOpts)
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "start failure") {
|
||||||
|
t.Fatalf("start-failure rerun err = %v, want start failure", err)
|
||||||
|
}
|
||||||
|
assertActivatedService(t, client, fixture, res)
|
||||||
|
if !bytes.Equal(before, installedBinarySHA(t, client, a)) {
|
||||||
|
t.Fatalf("installed binary changed after start-failure rerun")
|
||||||
|
}
|
||||||
|
assertNoActivationLeaks(t, client, fixture, a.DataDir)
|
||||||
|
|
||||||
|
// 4. Health failure preserves the prior install.
|
||||||
|
// WORKER_HOST=255.255.255.255 makes the new worker bind fail and
|
||||||
|
// /healthz unreachable; the process stays up but the health gate
|
||||||
|
// fails and the rollback restores the prior loopback deployment.
|
||||||
|
healthOpts := opts
|
||||||
|
healthOpts.Activation.Host = "255.255.255.255"
|
||||||
|
_, err = installer.SourceInstall(healthOpts)
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "health failure") {
|
||||||
|
t.Fatalf("health-failure rerun err = %v, want health failure", err)
|
||||||
|
}
|
||||||
|
assertActivatedService(t, client, fixture, res)
|
||||||
|
if !bytes.Equal(before, installedBinarySHA(t, client, a)) {
|
||||||
|
t.Fatalf("installed binary changed after health-failure rerun")
|
||||||
|
}
|
||||||
|
assertNoActivationLeaks(t, client, fixture, a.DataDir)
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestSourceInstallActivationNoStart installs the full layout (binary,
|
||||||
|
// env, data dir, service definition) without starting the worker, and
|
||||||
|
// asserts nothing runs afterward. Reruns stay idempotent.
|
||||||
|
//
|
||||||
|
// Opt-in: set RSMON_TEST_DOCKER=1 (see make test-ssh).
|
||||||
|
func TestSourceInstallActivationNoStart(t *testing.T) {
|
||||||
|
SkipUnlessEnabled(t)
|
||||||
|
|
||||||
|
const fixture = "alpine"
|
||||||
|
f := fixtureByName(t, fixture)
|
||||||
|
h, err := New("nostart-"+fixture, f)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute)
|
||||||
|
defer cancel()
|
||||||
|
if err := h.Start(ctx); err != nil {
|
||||||
|
t.Fatalf("start %s fixture: %v", fixture, err)
|
||||||
|
}
|
||||||
|
t.Cleanup(func() {
|
||||||
|
if err := h.Stop(); err != nil {
|
||||||
|
t.Errorf("stop: %v", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
opts := activationOptions(h, f)
|
||||||
|
opts.Activation.NoStart = true
|
||||||
|
res, err := installer.SourceInstall(opts)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("no-start install: %v", err)
|
||||||
|
}
|
||||||
|
client, err := h.Dial()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("dial: %v", err)
|
||||||
|
}
|
||||||
|
defer client.Close() //nolint:errcheck
|
||||||
|
a := res.Activation
|
||||||
|
if a == nil || a.Started {
|
||||||
|
t.Fatalf("no-start activation result = %+v", a)
|
||||||
|
}
|
||||||
|
if out, err := RunCommand(client, "test -x "+shellQuote(a.Binary)+" && test -f "+shellQuote(a.EnvFile)+" && echo INSTALLED"); err != nil || !strings.Contains(string(out), "INSTALLED") {
|
||||||
|
t.Fatalf("no-start did not install the layout: %q, %v", out, err)
|
||||||
|
}
|
||||||
|
if got := countWorkerProcesses(t, client, a.Binary); got != "0" {
|
||||||
|
t.Fatalf("no-start left a running worker: %q", got)
|
||||||
|
}
|
||||||
|
// A rerun is idempotent and still starts nothing.
|
||||||
|
if _, err := installer.SourceInstall(opts); err != nil {
|
||||||
|
t.Fatalf("no-start rerun: %v", err)
|
||||||
|
}
|
||||||
|
if got := countWorkerProcesses(t, client, a.Binary); got != "0" {
|
||||||
|
t.Fatalf("no-start rerun left a running worker: %q", got)
|
||||||
|
}
|
||||||
|
assertNoActivationLeaks(t, client, fixture, a.DataDir)
|
||||||
|
}
|
||||||
533
internal/installer/harness/integration_test.go
Обычный файл
533
internal/installer/harness/integration_test.go
Обычный файл
@@ -0,0 +1,533 @@
|
|||||||
|
package harness
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"crypto/ed25519"
|
||||||
|
"crypto/rand"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"golang.org/x/crypto/ssh"
|
||||||
|
"golang.org/x/crypto/ssh/knownhosts"
|
||||||
|
|
||||||
|
"rocketgit.ru/rsmon/worker/internal/installer"
|
||||||
|
"rocketgit.ru/rsmon/worker/internal/sshinstall"
|
||||||
|
)
|
||||||
|
|
||||||
|
// TestHarnessFixtures is the work-package-1 acceptance test: each distro
|
||||||
|
// fixture starts a real OpenSSH container, the harness waits for real
|
||||||
|
// network readiness, and the installer's Go SSH client connects, runs
|
||||||
|
// commands, and tears the environment down.
|
||||||
|
//
|
||||||
|
// Opt-in: set RSMON_TEST_DOCKER=1 (see make test-ssh).
|
||||||
|
func TestHarnessFixtures(t *testing.T) {
|
||||||
|
SkipUnlessEnabled(t)
|
||||||
|
|
||||||
|
for _, f := range Fixtures() {
|
||||||
|
f := f
|
||||||
|
t.Run(f.Name, func(t *testing.T) {
|
||||||
|
h, err := New("fixture-"+f.Name, f)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Minute)
|
||||||
|
defer cancel()
|
||||||
|
if err := h.Start(ctx); err != nil {
|
||||||
|
t.Fatalf("start %s fixture: %v", f.Name, err)
|
||||||
|
}
|
||||||
|
t.Cleanup(func() {
|
||||||
|
if err := h.Stop(); err != nil {
|
||||||
|
t.Errorf("stop %s fixture: %v", f.Name, err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
client, err := h.Dial()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("dial %s fixture: %v", f.Name, err)
|
||||||
|
}
|
||||||
|
defer client.Close() //nolint:errcheck
|
||||||
|
|
||||||
|
// 1. SSH command execution is real: round-trip a nonce.
|
||||||
|
nonce := fmt.Sprintf("RSMON_SSH_OK_%d", time.Now().UnixNano())
|
||||||
|
out, err := RunCommand(client, "printf '%s' "+shellQuote(nonce))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ssh round trip: %v", err)
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(string(out)) != nonce {
|
||||||
|
t.Fatalf("ssh round trip = %q, want %q", out, nonce)
|
||||||
|
}
|
||||||
|
|
||||||
|
// 2. The clean target has no Go toolchain and no worker source.
|
||||||
|
out, err = RunCommand(client, "command -v go || true; test ! -e /usr/local/go && echo NO_GO; test ! -e /opt/rsmon-worker-src && echo NO_SOURCE")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("clean-state probe: %v", err)
|
||||||
|
}
|
||||||
|
clean := string(out)
|
||||||
|
if strings.Contains(clean, "/go") && !strings.Contains(clean, "NO_GO") {
|
||||||
|
t.Fatalf("fixture unexpectedly has Go installed: %q", clean)
|
||||||
|
}
|
||||||
|
if !strings.Contains(clean, "NO_SOURCE") {
|
||||||
|
t.Fatalf("fixture unexpectedly has worker source: %q", clean)
|
||||||
|
}
|
||||||
|
|
||||||
|
// 3. Distro detection over the real session matches the fixture.
|
||||||
|
out, err = RunCommand(client, "cat /etc/os-release")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("read os-release: %v", err)
|
||||||
|
}
|
||||||
|
d := sshinstall.Detect(string(out), makeProber(client))
|
||||||
|
if d.Distro != f.Distro {
|
||||||
|
t.Fatalf("detected distro = %q, want %q (%s)", d.Distro, f.Distro, d.Summarize())
|
||||||
|
}
|
||||||
|
if d.PackageManager != f.Pkg {
|
||||||
|
t.Fatalf("detected package manager = %q, want %q", d.PackageManager, f.Pkg)
|
||||||
|
}
|
||||||
|
if d.InitSystem != f.Init {
|
||||||
|
t.Fatalf("detected init = %q, want %q", d.InitSystem, f.Init)
|
||||||
|
}
|
||||||
|
t.Logf("%s: %s", f.Name, d.Summarize())
|
||||||
|
|
||||||
|
// 4. A full source plan resolves for the detected host,
|
||||||
|
// including the pinned Go toolchain for its real arch.
|
||||||
|
out, err = RunCommand(client, "uname -m")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("uname -m: %v", err)
|
||||||
|
}
|
||||||
|
goarch, err := sshinstall.GoArch(strings.TrimSpace(string(out)))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("GoArch(%q): %v", out, err)
|
||||||
|
}
|
||||||
|
plan, err := sshinstall.PlanSource(d, sshinstall.SourceOptions{UnameM: strings.TrimSpace(string(out))})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("PlanSource: %v", err)
|
||||||
|
}
|
||||||
|
if plan.Toolchain.Arch != "linux-"+goarch {
|
||||||
|
t.Fatalf("plan toolchain %q does not match detected arch %q", plan.Toolchain.Arch, goarch)
|
||||||
|
}
|
||||||
|
if len(plan.Packages) == 0 || plan.Repo == "" {
|
||||||
|
t.Fatalf("incomplete plan: %+v", plan)
|
||||||
|
}
|
||||||
|
steps := plan.Steps()
|
||||||
|
if len(steps) != 6 {
|
||||||
|
t.Fatalf("plan steps = %d, want 6", len(steps))
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestHarnessHostKeyMismatch verifies the security gate: a dial against
|
||||||
|
// a known_hosts entry carrying a different host key must fail before any
|
||||||
|
// command can run.
|
||||||
|
func TestHarnessHostKeyMismatch(t *testing.T) {
|
||||||
|
SkipUnlessEnabled(t)
|
||||||
|
|
||||||
|
f := Fixtures()[0] // alpine is the smallest fixture
|
||||||
|
h, err := New("hostkey-"+f.Name, f)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Minute)
|
||||||
|
defer cancel()
|
||||||
|
if err := h.Start(ctx); err != nil {
|
||||||
|
t.Fatalf("start fixture: %v", err)
|
||||||
|
}
|
||||||
|
t.Cleanup(func() {
|
||||||
|
if err := h.Stop(); err != nil {
|
||||||
|
t.Errorf("stop fixture: %v", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
// Build a known_hosts entry with a different (freshly generated) key.
|
||||||
|
_, priv, err := ed25519.GenerateKey(rand.Reader)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
signer, err := ssh.NewSignerFromKey(priv)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
wrongFile := filepath.Join(t.TempDir(), "known_hosts")
|
||||||
|
line := knownhosts.Line([]string{h.Addr()}, signer.PublicKey())
|
||||||
|
if err := os.WriteFile(wrongFile, []byte(line+"\n"), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
keyBytes, err := os.ReadFile(testKeyPath())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
keySigner, err := ssh.ParsePrivateKey(keyBytes)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
callback, err := knownhosts.New(wrongFile)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
config := &ssh.ClientConfig{
|
||||||
|
User: f.UserOrDefault(),
|
||||||
|
Auth: []ssh.AuthMethod{ssh.PublicKeys(keySigner)},
|
||||||
|
HostKeyCallback: callback,
|
||||||
|
Timeout: 15 * time.Second,
|
||||||
|
}
|
||||||
|
client, err := ssh.Dial("tcp", h.Addr(), config)
|
||||||
|
if err == nil {
|
||||||
|
client.Close() //nolint:errcheck
|
||||||
|
t.Fatal("dial with a mismatched host key succeeded")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "knownhosts") && !strings.Contains(err.Error(), "key") {
|
||||||
|
t.Fatalf("host-key mismatch error = %v, want a key/host verification failure", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestHarnessTeardown verifies Stop reliably removes the container, the
|
||||||
|
// dedicated network, the per-instance fixture image tag (never a shared
|
||||||
|
// base image), and the temp known_hosts directory.
|
||||||
|
func TestHarnessTeardown(t *testing.T) {
|
||||||
|
SkipUnlessEnabled(t)
|
||||||
|
|
||||||
|
f := Fixtures()[0]
|
||||||
|
h, err := New("teardown-"+f.Name, f)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Minute)
|
||||||
|
defer cancel()
|
||||||
|
if err := h.Start(ctx); err != nil {
|
||||||
|
t.Fatalf("start fixture: %v", err)
|
||||||
|
}
|
||||||
|
if h.ContainerID() == "" {
|
||||||
|
t.Fatal("container id empty after start")
|
||||||
|
}
|
||||||
|
if h.KnownHostsPath() == "" {
|
||||||
|
t.Fatal("known_hosts not created after start")
|
||||||
|
}
|
||||||
|
knownHostsDir := filepath.Dir(h.KnownHostsPath())
|
||||||
|
if err := h.Stop(); err != nil {
|
||||||
|
t.Fatalf("stop: %v", err)
|
||||||
|
}
|
||||||
|
if _, err := h.DockerCmd(ctx, "inspect", h.ContainerName()); err == nil {
|
||||||
|
t.Fatal("container still present after Stop")
|
||||||
|
}
|
||||||
|
if _, err := h.DockerCmd(ctx, "network", "inspect", h.NetworkName()); err == nil {
|
||||||
|
t.Fatal("network still present after Stop")
|
||||||
|
}
|
||||||
|
if _, err := h.DockerCmd(ctx, "image", "inspect", h.ImageTag()); err == nil {
|
||||||
|
t.Fatalf("fixture image tag %q still present after Stop", h.ImageTag())
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(knownHostsDir); !os.IsNotExist(err) {
|
||||||
|
t.Fatalf("known_hosts temp dir %q still present after Stop: %v", knownHostsDir, err)
|
||||||
|
}
|
||||||
|
if h.KnownHostsPath() != "" {
|
||||||
|
t.Fatalf("KnownHostsPath = %q after Stop, want empty", h.KnownHostsPath())
|
||||||
|
}
|
||||||
|
// Stop is idempotent.
|
||||||
|
if err := h.Stop(); err != nil {
|
||||||
|
t.Fatalf("second stop: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestHarnessHostKeyStableAcrossDial ensures the host key captured at
|
||||||
|
// readiness is the one verified on every later dial, so a successful
|
||||||
|
// Dial is proof of verified, real SSH transport.
|
||||||
|
func TestHarnessHostKeyStable(t *testing.T) {
|
||||||
|
SkipUnlessEnabled(t)
|
||||||
|
|
||||||
|
f := Fixtures()[0]
|
||||||
|
h, err := New("key-"+f.Name, f)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Minute)
|
||||||
|
defer cancel()
|
||||||
|
if err := h.Start(ctx); err != nil {
|
||||||
|
t.Fatalf("start fixture: %v", err)
|
||||||
|
}
|
||||||
|
t.Cleanup(func() {
|
||||||
|
if err := h.Stop(); err != nil {
|
||||||
|
t.Errorf("stop fixture: %v", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
client, err := h.Dial()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("dial: %v", err)
|
||||||
|
}
|
||||||
|
client.Close() //nolint:errcheck
|
||||||
|
client, err = h.Dial()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("second dial: %v", err)
|
||||||
|
}
|
||||||
|
defer client.Close() //nolint:errcheck
|
||||||
|
if out, err := RunCommand(client, "echo VERIFIED"); err != nil || strings.TrimSpace(string(out)) != "VERIFIED" {
|
||||||
|
t.Fatalf("verified session command = %q, %v", out, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestSourceInstallFixtures is the work-package-3 acceptance test: each
|
||||||
|
// distro fixture starts clean (no Go, no worker source) and the real
|
||||||
|
// installer executes the full source flow over SSH - prerequisite
|
||||||
|
// install, verified Go toolchain download/extraction, clone/update of the
|
||||||
|
// public repository, resolved branch/commit record, and a build to a
|
||||||
|
// staging path. The running service and its config are deliberately not
|
||||||
|
// installed (that is work package 4). A rerun exercises idempotency.
|
||||||
|
//
|
||||||
|
// Opt-in: set RSMON_TEST_DOCKER=1 (see make test-ssh).
|
||||||
|
func TestSourceInstallFixtures(t *testing.T) {
|
||||||
|
SkipUnlessEnabled(t)
|
||||||
|
|
||||||
|
for _, f := range Fixtures() {
|
||||||
|
f := f
|
||||||
|
t.Run(f.Name, func(t *testing.T) {
|
||||||
|
h, err := New("source-"+f.Name, f)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute)
|
||||||
|
defer cancel()
|
||||||
|
if err := h.Start(ctx); err != nil {
|
||||||
|
t.Fatalf("start %s fixture: %v", f.Name, err)
|
||||||
|
}
|
||||||
|
t.Cleanup(func() {
|
||||||
|
if err := h.Stop(); err != nil {
|
||||||
|
t.Errorf("stop %s fixture: %v", f.Name, err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
// Clean state: no Go toolchain, no source, no leftover
|
||||||
|
// toolchain temp dirs.
|
||||||
|
client, err := h.Dial()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("dial %s fixture: %v", f.Name, err)
|
||||||
|
}
|
||||||
|
probe, err := RunCommand(client, "command -v go || true; test ! -e /usr/local/go && echo NO_GO; test ! -e /opt/rsmon-worker-src && echo NO_SOURCE; ls /tmp | grep -q rsmon-toolchain && echo LEAK; ls /usr/local 2>/dev/null | grep -qE 'go-staging|go-backup' && echo LEAK; echo DONE")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("clean-state probe: %v", err)
|
||||||
|
}
|
||||||
|
clean := string(probe)
|
||||||
|
if !strings.Contains(clean, "NO_GO") || !strings.Contains(clean, "NO_SOURCE") {
|
||||||
|
t.Fatalf("fixture is not clean: %q", clean)
|
||||||
|
}
|
||||||
|
if strings.Contains(clean, "LEAK") {
|
||||||
|
t.Fatalf("fixture has leftover toolchain temp dirs: %q", clean)
|
||||||
|
}
|
||||||
|
client.Close() //nolint:errcheck
|
||||||
|
|
||||||
|
opts := installer.SourceInstallOptions{
|
||||||
|
SSHOptions: installer.SSHOptions{
|
||||||
|
Host: "127.0.0.1",
|
||||||
|
Port: h.Port(),
|
||||||
|
User: f.UserOrDefault(),
|
||||||
|
IdentityFile: testKeyPath(),
|
||||||
|
KnownHostsFile: h.KnownHostsPath(),
|
||||||
|
},
|
||||||
|
// Staging-only: service activation (work package 4) is
|
||||||
|
// exercised by the dedicated activation tests.
|
||||||
|
Activation: installer.ActivationOptions{Activate: false},
|
||||||
|
}
|
||||||
|
if repo := strings.TrimSpace(os.Getenv("RSMON_TEST_SOURCE_REPO")); repo != "" {
|
||||||
|
opts.Repo = repo
|
||||||
|
}
|
||||||
|
if branch := strings.TrimSpace(os.Getenv("RSMON_TEST_SOURCE_BRANCH")); branch != "" {
|
||||||
|
opts.Branch = branch
|
||||||
|
}
|
||||||
|
|
||||||
|
res, err := installer.SourceInstall(opts)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("source install on %s: %v", f.Name, err)
|
||||||
|
}
|
||||||
|
t.Logf("%s: %s -> branch=%s commit=%s staged=%s", f.Name, res.Detection.Summarize(),
|
||||||
|
res.ResolvedBranch, res.ResolvedCommit, res.StageBinary)
|
||||||
|
|
||||||
|
if res.Detection.Distro != f.Distro || res.Detection.PackageManager != f.Pkg || res.Detection.InitSystem != f.Init {
|
||||||
|
t.Fatalf("detection = %+v, want %s/%s/%s", res.Detection, f.Distro, f.Pkg, f.Init)
|
||||||
|
}
|
||||||
|
if res.GoArch != "linux-"+strings.TrimPrefix(res.Plan.Toolchain.Arch, "linux-") {
|
||||||
|
t.Fatalf("resolved arch = %q, want %q", res.GoArch, res.Plan.Toolchain.Arch)
|
||||||
|
}
|
||||||
|
if res.ResolvedBranch == "" || len(res.ResolvedCommit) != 40 {
|
||||||
|
t.Fatalf("resolved branch/commit incomplete: %s @ %s", res.ResolvedBranch, res.ResolvedCommit)
|
||||||
|
}
|
||||||
|
if res.StageBinary == "" || res.RecordFile == "" || res.ToolchainDir == "" {
|
||||||
|
t.Fatalf("result paths incomplete: %+v", res)
|
||||||
|
}
|
||||||
|
|
||||||
|
client, err = h.Dial()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("redial: %v", err)
|
||||||
|
}
|
||||||
|
defer client.Close() //nolint:errcheck
|
||||||
|
assertSourceInstallState(t, client, f.Name, res)
|
||||||
|
|
||||||
|
// Idempotent rerun: succeeds, resolves the same branch,
|
||||||
|
// reuses the toolchain, and leaks no temp files.
|
||||||
|
res2, err := installer.SourceInstall(opts)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("source install rerun on %s: %v", f.Name, err)
|
||||||
|
}
|
||||||
|
if res2.ResolvedBranch != res.ResolvedBranch || len(res2.ResolvedCommit) != 40 {
|
||||||
|
t.Fatalf("rerun resolved = %s @ %s, want branch %s", res2.ResolvedBranch, res2.ResolvedCommit, res.ResolvedBranch)
|
||||||
|
}
|
||||||
|
assertSourceInstallState(t, client, f.Name, res2)
|
||||||
|
if out, err := RunCommand(client, "leak=0; ls /tmp | grep -q rsmon-toolchain && leak=1; ls /usr/local 2>/dev/null | grep -qE 'go-staging|go-backup' && leak=1; [ \"$leak\" -eq 1 ] && echo LEAK || echo CLEAN"); err != nil || strings.TrimSpace(string(out)) != "CLEAN" {
|
||||||
|
t.Fatalf("toolchain temp dirs leaked after rerun: %q, %v", out, err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// assertSourceInstallState verifies the remote side-effects of a source
|
||||||
|
// install: the toolchain reports the pinned version, the staging binary
|
||||||
|
// exists and reports the resolved commit, and the record file carries
|
||||||
|
// the resolved branch and commit.
|
||||||
|
func assertSourceInstallState(t *testing.T, client *ssh.Client, name string, res *installer.SourceInstallResult) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
out, err := RunCommand(client, res.ToolchainDir+"/bin/go version")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("%s: toolchain missing: %v", name, err)
|
||||||
|
}
|
||||||
|
if !strings.Contains(string(out), "go"+res.Plan.Toolchain.Version) {
|
||||||
|
t.Fatalf("%s: toolchain version = %q, want go%s", name, out, res.Plan.Toolchain.Version)
|
||||||
|
}
|
||||||
|
|
||||||
|
out, err = RunCommand(client, "test -x "+shellQuote(res.StageBinary)+" && echo BUILT")
|
||||||
|
if err != nil || !strings.Contains(string(out), "BUILT") {
|
||||||
|
t.Fatalf("%s: staging binary not present at %s: %q, %v", name, res.StageBinary, out, err)
|
||||||
|
}
|
||||||
|
out, err = RunCommand(client, shellQuote(res.StageBinary)+" --version")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("%s: staging binary --version: %v", name, err)
|
||||||
|
}
|
||||||
|
if !strings.Contains(string(out), "commit="+res.ResolvedCommit[:12]) {
|
||||||
|
t.Fatalf("%s: staging binary reports commit %q, want short %s", name, out, res.ResolvedCommit[:12])
|
||||||
|
}
|
||||||
|
|
||||||
|
record, err := RunCommand(client, "cat "+shellQuote(res.RecordFile))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("%s: read record file: %v", name, err)
|
||||||
|
}
|
||||||
|
if !strings.Contains(string(record), "branch="+res.ResolvedBranch) || !strings.Contains(string(record), "commit="+res.ResolvedCommit) {
|
||||||
|
t.Fatalf("%s: record file = %q, want branch=%s commit=%s", name, record, res.ResolvedBranch, res.ResolvedCommit)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestSourceInstallDirtyCheckoutPreservesStaging is the work-package-3
|
||||||
|
// failure-atomicity test: after a successful install, dirtying the
|
||||||
|
// tracked working tree makes the next checkout fail closed. The rerun
|
||||||
|
// must report the checkout error without ever reaching the build step,
|
||||||
|
// leaving the previous staging binary and commit record byte-for-byte
|
||||||
|
// unchanged, and without leaking toolchain staging/backup directories.
|
||||||
|
func TestSourceInstallDirtyCheckoutPreservesStaging(t *testing.T) {
|
||||||
|
SkipUnlessEnabled(t)
|
||||||
|
|
||||||
|
f := Fixtures()[0] // alpine is the smallest fixture
|
||||||
|
h, err := New("dirty-"+f.Name, f)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute)
|
||||||
|
defer cancel()
|
||||||
|
if err := h.Start(ctx); err != nil {
|
||||||
|
t.Fatalf("start %s fixture: %v", f.Name, err)
|
||||||
|
}
|
||||||
|
t.Cleanup(func() {
|
||||||
|
if err := h.Stop(); err != nil {
|
||||||
|
t.Errorf("stop %s fixture: %v", f.Name, err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
opts := installer.SourceInstallOptions{
|
||||||
|
SSHOptions: installer.SSHOptions{
|
||||||
|
Host: "127.0.0.1",
|
||||||
|
Port: h.Port(),
|
||||||
|
User: f.UserOrDefault(),
|
||||||
|
IdentityFile: testKeyPath(),
|
||||||
|
KnownHostsFile: h.KnownHostsPath(),
|
||||||
|
},
|
||||||
|
Activation: installer.ActivationOptions{Activate: false},
|
||||||
|
}
|
||||||
|
if repo := strings.TrimSpace(os.Getenv("RSMON_TEST_SOURCE_REPO")); repo != "" {
|
||||||
|
opts.Repo = repo
|
||||||
|
}
|
||||||
|
res, err := installer.SourceInstall(opts)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("initial source install: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
client, err := h.Dial()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("dial: %v", err)
|
||||||
|
}
|
||||||
|
defer client.Close() //nolint:errcheck
|
||||||
|
|
||||||
|
// Dirty a tracked file so the rerun's checkout refuses to proceed.
|
||||||
|
// Appending a marker is deterministic regardless of the cloned history
|
||||||
|
// (a `git checkout master~1 -- <file>` would depend on whether that
|
||||||
|
// commit differs from the resolved one), and an appended line to a
|
||||||
|
// tracked file is always an unstaged modification `git diff --quiet`
|
||||||
|
// catches.
|
||||||
|
if _, err := RunCommand(client, "printf '\\n# rsmon-worker dirty-tree marker\\n' >> "+shellQuote(res.Plan.BuildDir)+"/Makefile"); err != nil {
|
||||||
|
t.Fatalf("dirty the working tree: %v", err)
|
||||||
|
}
|
||||||
|
beforeBinary, err := RunCommand(client, "sha256sum "+shellQuote(res.StageBinary))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
beforeRecord, err := RunCommand(client, "cat "+shellQuote(res.RecordFile))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if _, err := installer.SourceInstall(opts); err == nil || !strings.Contains(err.Error(), "check out branch") {
|
||||||
|
t.Fatalf("rerun err = %v, want checkout failure on dirty tree", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
afterBinary, err := RunCommand(client, "sha256sum "+shellQuote(res.StageBinary))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
afterRecord, err := RunCommand(client, "cat "+shellQuote(res.RecordFile))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !bytes.Equal(bytes.TrimSpace(beforeBinary), bytes.TrimSpace(afterBinary)) {
|
||||||
|
t.Fatalf("staging binary changed after failed rerun:\nbefore: %s\nafter: %s", beforeBinary, afterBinary)
|
||||||
|
}
|
||||||
|
if !bytes.Equal(bytes.TrimSpace(beforeRecord), bytes.TrimSpace(afterRecord)) {
|
||||||
|
t.Fatalf("commit record changed after failed rerun:\nbefore: %s\nafter: %s", beforeRecord, afterRecord)
|
||||||
|
}
|
||||||
|
if out, err := RunCommand(client, "leak=0; ls /tmp | grep -q rsmon-toolchain && leak=1; ls /usr/local 2>/dev/null | grep -qE 'go-staging|go-backup' && leak=1; [ \"$leak\" -eq 1 ] && echo LEAK || echo CLEAN"); err != nil || strings.TrimSpace(string(out)) != "CLEAN" {
|
||||||
|
t.Fatalf("toolchain staging leaked after failed rerun: %q, %v", out, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// makeProber builds an sshinstall.FileProber over a live SSH session.
|
||||||
|
func makeProber(client *ssh.Client) sshinstall.FileProber {
|
||||||
|
return func(paths ...string) map[string]bool {
|
||||||
|
// The trailing `; true` keeps the shell exit status 0: the
|
||||||
|
// last `[ -e "$p" ]` in the loop would otherwise set exit 1
|
||||||
|
// when the final path is absent (as on Arch), which is not an
|
||||||
|
// error for a probe.
|
||||||
|
expr := "for p in " + strings.Join(paths, " ") + "; do [ -e \"$p\" ] && printf '%s\\n' \"$p\"; done; true"
|
||||||
|
out, err := RunCommand(client, expr)
|
||||||
|
if err != nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
present := make(map[string]bool, len(paths))
|
||||||
|
for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") {
|
||||||
|
if line = strings.TrimSpace(line); line != "" {
|
||||||
|
present[line] = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return present
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func shellQuote(value string) string {
|
||||||
|
return "'" + strings.ReplaceAll(value, "'", "'\\''") + "'"
|
||||||
|
}
|
||||||
21
internal/installer/harness/testdata/fixtures/alpine/Dockerfile
поставляемый
Обычный файл
21
internal/installer/harness/testdata/fixtures/alpine/Dockerfile
поставляемый
Обычный файл
@@ -0,0 +1,21 @@
|
|||||||
|
# Alpine OpenSSH fixture for the source-install harness.
|
||||||
|
#
|
||||||
|
# Base image is the reg.rsxx.ru mirror (alpine:3). openrc is installed
|
||||||
|
# explicitly so init-system detection has a stable marker; openssh is the
|
||||||
|
# minimal sshd. Password auth is disabled; the fixture authenticates with
|
||||||
|
# the shared test key.
|
||||||
|
FROM reg.rsxx.ru/library/alpine:3
|
||||||
|
|
||||||
|
RUN apk add --no-cache openssh openrc \
|
||||||
|
&& mkdir -p /run/sshd /root/.ssh \
|
||||||
|
&& chmod 700 /root/.ssh \
|
||||||
|
&& ssh-keygen -A \
|
||||||
|
&& sed -i 's/^#\?PermitRootLogin.*/PermitRootLogin prohibit-password/' /etc/ssh/sshd_config \
|
||||||
|
&& sed -i 's/^#\?PasswordAuthentication.*/PasswordAuthentication no/' /etc/ssh/sshd_config \
|
||||||
|
&& sed -i 's/^#\?PubkeyAuthentication.*/PubkeyAuthentication yes/' /etc/ssh/sshd_config
|
||||||
|
|
||||||
|
COPY keys/rsmon_test_ed25519.pub /root/.ssh/authorized_keys
|
||||||
|
RUN chmod 600 /root/.ssh/authorized_keys
|
||||||
|
|
||||||
|
EXPOSE 22
|
||||||
|
CMD ["/usr/sbin/sshd", "-D", "-e"]
|
||||||
19
internal/installer/harness/testdata/fixtures/arch/Dockerfile
поставляемый
Обычный файл
19
internal/installer/harness/testdata/fixtures/arch/Dockerfile
поставляемый
Обычный файл
@@ -0,0 +1,19 @@
|
|||||||
|
# Arch Linux OpenSSH fixture for the source-install harness.
|
||||||
|
#
|
||||||
|
# No reg.rsxx.ru mirror exists for Arch yet, so the default is the
|
||||||
|
# Docker Hub archlinux:latest image. Override with RSMON_TEST_IMAGE_ARCH.
|
||||||
|
FROM archlinux:latest
|
||||||
|
|
||||||
|
RUN pacman -Sy --noconfirm --needed openssh \
|
||||||
|
&& mkdir -p /run/sshd /root/.ssh \
|
||||||
|
&& chmod 700 /root/.ssh \
|
||||||
|
&& ssh-keygen -A \
|
||||||
|
&& sed -i 's/^#\?PermitRootLogin.*/PermitRootLogin prohibit-password/' /etc/ssh/sshd_config \
|
||||||
|
&& sed -i 's/^#\?PasswordAuthentication.*/PasswordAuthentication no/' /etc/ssh/sshd_config \
|
||||||
|
&& sed -i 's/^#\?PubkeyAuthentication.*/PubkeyAuthentication yes/' /etc/ssh/sshd_config
|
||||||
|
|
||||||
|
COPY keys/rsmon_test_ed25519.pub /root/.ssh/authorized_keys
|
||||||
|
RUN chmod 600 /root/.ssh/authorized_keys
|
||||||
|
|
||||||
|
EXPOSE 22
|
||||||
|
CMD ["/usr/sbin/sshd", "-D", "-e"]
|
||||||
20
internal/installer/harness/testdata/fixtures/ubuntu/Dockerfile
поставляемый
Обычный файл
20
internal/installer/harness/testdata/fixtures/ubuntu/Dockerfile
поставляемый
Обычный файл
@@ -0,0 +1,20 @@
|
|||||||
|
# Ubuntu OpenSSH fixture for the source-install harness.
|
||||||
|
#
|
||||||
|
# No reg.rsxx.ru mirror exists for Ubuntu yet, so the default is the
|
||||||
|
# Docker Hub ubuntu:24.04 image. Override with RSMON_TEST_IMAGE_UBUNTU.
|
||||||
|
FROM ubuntu:24.04
|
||||||
|
|
||||||
|
RUN apt-get update \
|
||||||
|
&& DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends openssh-server \
|
||||||
|
&& rm -rf /var/lib/apt/lists/* \
|
||||||
|
&& mkdir -p /run/sshd /root/.ssh \
|
||||||
|
&& chmod 700 /root/.ssh \
|
||||||
|
&& sed -i 's/^#\?PermitRootLogin.*/PermitRootLogin prohibit-password/' /etc/ssh/sshd_config \
|
||||||
|
&& sed -i 's/^#\?PasswordAuthentication.*/PasswordAuthentication no/' /etc/ssh/sshd_config \
|
||||||
|
&& sed -i 's/^#\?PubkeyAuthentication.*/PubkeyAuthentication yes/' /etc/ssh/sshd_config
|
||||||
|
|
||||||
|
COPY keys/rsmon_test_ed25519.pub /root/.ssh/authorized_keys
|
||||||
|
RUN chmod 600 /root/.ssh/authorized_keys
|
||||||
|
|
||||||
|
EXPOSE 22
|
||||||
|
CMD ["/usr/sbin/sshd", "-D", "-e"]
|
||||||
8
internal/installer/harness/testdata/keys/rsmon_test_ed25519
поставляемый
Обычный файл
8
internal/installer/harness/testdata/keys/rsmon_test_ed25519
поставляемый
Обычный файл
@@ -0,0 +1,8 @@
|
|||||||
|
-----BEGIN OPENSSH PRIVATE KEY-----
|
||||||
|
b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAMwAAAAtzc2gtZW
|
||||||
|
QyNTUxOQAAACCz7h3HFwJrC+DIrE3W+9yI+hRAcCVesyEQmCicmPrqAwAAAKgIO12YCDtd
|
||||||
|
mAAAAAtzc2gtZWQyNTUxOQAAACCz7h3HFwJrC+DIrE3W+9yI+hRAcCVesyEQmCicmPrqAw
|
||||||
|
AAAEDqbSMcuhF56miNJOZKUOuA/9I6yVga06nirb7pns41lLPuHccXAmsL4MisTdb73Ij6
|
||||||
|
FEBwJV6zIRCYKJyY+uoDAAAAIHJzbW9uLXdvcmtlciBzb3VyY2UtaW5zdGFsbCB0ZXN0AQ
|
||||||
|
IDBAU=
|
||||||
|
-----END OPENSSH PRIVATE KEY-----
|
||||||
1
internal/installer/harness/testdata/keys/rsmon_test_ed25519.pub
поставляемый
Обычный файл
1
internal/installer/harness/testdata/keys/rsmon_test_ed25519.pub
поставляемый
Обычный файл
@@ -0,0 +1 @@
|
|||||||
|
ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAILPuHccXAmsL4MisTdb73Ij6FEBwJV6zIRCYKJyY+uoD rsmon-worker source-install test
|
||||||
608
internal/installer/install.go
Обычный файл
608
internal/installer/install.go
Обычный файл
@@ -0,0 +1,608 @@
|
|||||||
|
package installer
|
||||||
|
|
||||||
|
import (
|
||||||
|
"crypto/rand"
|
||||||
|
"encoding/hex"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"net/url"
|
||||||
|
"os"
|
||||||
|
"os/exec"
|
||||||
|
"path/filepath"
|
||||||
|
"regexp"
|
||||||
|
"strings"
|
||||||
|
"unicode"
|
||||||
|
|
||||||
|
"github.com/joho/godotenv"
|
||||||
|
|
||||||
|
"rocketgit.ru/rsmon/worker/internal/distworker"
|
||||||
|
)
|
||||||
|
|
||||||
|
var (
|
||||||
|
dockerImagePattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._/:-]*@sha256:[a-f0-9]{64}$`)
|
||||||
|
envKeyPattern = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*$`)
|
||||||
|
instanceNamePattern = regexp.MustCompile(`^[a-z0-9]([a-z0-9-]*[a-z0-9])?$`)
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
DefaultURL = "https://rsmon.ru"
|
||||||
|
DefaultImage = ""
|
||||||
|
|
||||||
|
defaultLogin = "admin"
|
||||||
|
defaultPort = "27401"
|
||||||
|
)
|
||||||
|
|
||||||
|
// installEnvKeys is the canonical, ordered set of worker environment
|
||||||
|
// variables the installer understands and writes to the unit's env
|
||||||
|
// file. Order matters: the rendered file is stable and readable.
|
||||||
|
//
|
||||||
|
// PUBLIC_URL is the canonical advertised origin. WORKER_URL stays in the
|
||||||
|
// list for the bounded migration so legacy env files still resolve; it
|
||||||
|
// is dropped from the written file whenever PUBLIC_URL is also present.
|
||||||
|
var installEnvKeys = []string{
|
||||||
|
"RSMON_URL",
|
||||||
|
"RSMON_TOKEN",
|
||||||
|
"WORKER_HOST",
|
||||||
|
"WORKER_PORT",
|
||||||
|
"PUBLIC_URL",
|
||||||
|
"WORKER_URL",
|
||||||
|
"WORKER_LOGIN",
|
||||||
|
"WORKER_PASSWORD",
|
||||||
|
"WORKER_COMPOSE_ENABLED",
|
||||||
|
"WORKER_CLUSTER_ENABLED",
|
||||||
|
"WORKER_CLUSTER_ID",
|
||||||
|
"WORKER_CLUSTER_PORT",
|
||||||
|
"WORKER_CLUSTER_PEERS",
|
||||||
|
"WORKER_CLUSTER_DATA_DIR",
|
||||||
|
"WORKER_RELEASE_URL",
|
||||||
|
}
|
||||||
|
|
||||||
|
// InstallOptions captures the install/deploy command-line knobs. The
|
||||||
|
// classic single-instance install leaves Name empty; a non-empty Name
|
||||||
|
// installs a co-located worker under rsmon-worker-<name> with its own
|
||||||
|
// binary path, config dir, data dir, systemd unit, and port.
|
||||||
|
type InstallOptions struct {
|
||||||
|
Binary string
|
||||||
|
EnvFile string
|
||||||
|
Token string
|
||||||
|
URL string
|
||||||
|
PublicURL string
|
||||||
|
Host string
|
||||||
|
Port string
|
||||||
|
Login string
|
||||||
|
Password string
|
||||||
|
Name string
|
||||||
|
Docker bool
|
||||||
|
Image string
|
||||||
|
NoStart bool
|
||||||
|
}
|
||||||
|
|
||||||
|
// paths is the fully-resolved on-disk layout for an instance. Every
|
||||||
|
// installer write targets these. The empty Name yields the classic
|
||||||
|
// rsmon-worker paths; a named instance appends -<name> everywhere.
|
||||||
|
type paths struct {
|
||||||
|
name string
|
||||||
|
binary string // /usr/local/bin/rsmon-worker[-name]
|
||||||
|
configDir string // /etc/rsmon-worker[-name]
|
||||||
|
envFile string // <configDir>/worker.env
|
||||||
|
dataDir string // /var/lib/rsmon-worker[-name]
|
||||||
|
unitName string // rsmon-worker[-name].service
|
||||||
|
unitFile string // /etc/systemd/system/<unitName>
|
||||||
|
container string // docker container name (rsmon-worker[-name])
|
||||||
|
volume string // docker volume name (rsmon-worker-data[-name])
|
||||||
|
}
|
||||||
|
|
||||||
|
// resolvePaths derives the on-disk layout for an instance name. The
|
||||||
|
// empty name reproduces the legacy single-instance paths so existing
|
||||||
|
// deployments keep upgrading in place.
|
||||||
|
func resolvePaths(name string) paths {
|
||||||
|
p := paths{
|
||||||
|
name: name,
|
||||||
|
binary: "/usr/local/bin/rsmon-worker",
|
||||||
|
configDir: "/etc/rsmon-worker",
|
||||||
|
dataDir: "/var/lib/rsmon-worker",
|
||||||
|
container: "rsmon-worker",
|
||||||
|
volume: "rsmon-worker-data",
|
||||||
|
}
|
||||||
|
if name == "" {
|
||||||
|
p.unitName = "rsmon-worker.service"
|
||||||
|
} else {
|
||||||
|
suffix := "-" + name
|
||||||
|
p.binary += suffix
|
||||||
|
p.configDir += suffix
|
||||||
|
p.dataDir += suffix
|
||||||
|
p.container += suffix
|
||||||
|
p.volume += suffix
|
||||||
|
p.unitName = "rsmon-worker" + suffix + ".service"
|
||||||
|
}
|
||||||
|
p.envFile = filepath.Join(p.configDir, "worker.env")
|
||||||
|
p.unitFile = "/etc/systemd/system/" + p.unitName
|
||||||
|
return p
|
||||||
|
}
|
||||||
|
|
||||||
|
// validateInstanceName allows lowercase alphanumeric and hyphens, 1-32
|
||||||
|
// chars, starting and ending alphanumeric. The empty string (the
|
||||||
|
// primary instance) is always valid.
|
||||||
|
func validateInstanceName(name string) error {
|
||||||
|
if name == "" {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if len(name) > 32 || !instanceNamePattern.MatchString(name) {
|
||||||
|
return errors.New("instance name must be 1-32 chars, lowercase alphanumeric and hyphens, starting and ending alphanumeric")
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Install copies the worker binary (or pulls the Docker image), writes
|
||||||
|
// a hardened per-instance systemd unit and env file, then enables and
|
||||||
|
// (unless --no-start) starts the service.
|
||||||
|
func Install(opts InstallOptions) error {
|
||||||
|
if os.Geteuid() != 0 {
|
||||||
|
return errors.New("install must be run as root")
|
||||||
|
}
|
||||||
|
name := strings.TrimSpace(opts.Name)
|
||||||
|
if err := validateInstanceName(name); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
p := resolvePaths(name)
|
||||||
|
|
||||||
|
var fileEnv map[string]string
|
||||||
|
if opts.EnvFile != "" {
|
||||||
|
// ValidateEnvironmentFile is the strict systemd-safe gate; it
|
||||||
|
// also confirms RSMON_URL/RSMON_TOKEN are present in the file.
|
||||||
|
if err := ValidateEnvironmentFile(opts.EnvFile); err != nil {
|
||||||
|
return fmt.Errorf("--env-file: %w", err)
|
||||||
|
}
|
||||||
|
fe, err := godotenv.Read(opts.EnvFile)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("read --env-file: %w", err)
|
||||||
|
}
|
||||||
|
fileEnv = fe
|
||||||
|
}
|
||||||
|
|
||||||
|
values, err := resolveInstallEnv(opts, name, fileEnv)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
generatedPassword := ""
|
||||||
|
if values["WORKER_LOGIN"] == "" && values["WORKER_PASSWORD"] == "" {
|
||||||
|
gen, gerr := generatePassword(18)
|
||||||
|
if gerr != nil {
|
||||||
|
return gerr
|
||||||
|
}
|
||||||
|
values["WORKER_LOGIN"] = defaultLogin
|
||||||
|
values["WORKER_PASSWORD"] = gen
|
||||||
|
generatedPassword = gen
|
||||||
|
}
|
||||||
|
|
||||||
|
// Stage the binary (or pull the image) before touching config so a
|
||||||
|
// download/build failure leaves the host untouched.
|
||||||
|
if opts.Docker {
|
||||||
|
if err := ValidateImage(opts.Image); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if _, err := exec.LookPath("docker"); err != nil {
|
||||||
|
return errors.New("docker is required for --docker installation")
|
||||||
|
}
|
||||||
|
if err := command("docker", "pull", opts.Image); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
binary := opts.Binary
|
||||||
|
if binary == "" {
|
||||||
|
ex, err := os.Executable()
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("locate worker executable: %w", err)
|
||||||
|
}
|
||||||
|
binary = ex
|
||||||
|
}
|
||||||
|
if err := copyAtomic(binary, p.binary, 0755); err != nil {
|
||||||
|
return fmt.Errorf("install binary: %w", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := os.MkdirAll(filepath.Join(p.dataDir, "webapp"), 0755); err != nil {
|
||||||
|
return fmt.Errorf("create data dir: %w", err)
|
||||||
|
}
|
||||||
|
if err := os.MkdirAll(p.configDir, 0750); err != nil {
|
||||||
|
return fmt.Errorf("create config dir: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
unit := systemdUnitFor(p)
|
||||||
|
if opts.Docker {
|
||||||
|
unit = dockerUnitFor(p, opts.Image)
|
||||||
|
}
|
||||||
|
if err := writeAtomic(p.unitFile, []byte(unit), 0644); err != nil {
|
||||||
|
return fmt.Errorf("install systemd unit: %w", err)
|
||||||
|
}
|
||||||
|
if err := writeAtomic(p.envFile, renderEnvFile(values), 0600); err != nil {
|
||||||
|
return fmt.Errorf("install environment: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := command("systemctl", "daemon-reload"); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := command("systemctl", "enable", p.unitName); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if !opts.NoStart {
|
||||||
|
if err := command("systemctl", "restart", p.unitName); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := command("systemctl", "is-active", "--quiet", p.unitName); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Operator-facing summary. Keep it on stdout so it composes with
|
||||||
|
// scripts; the generated password is only printed once, here.
|
||||||
|
fmt.Printf("rsmon-worker installed: %s\n", p.unitName)
|
||||||
|
fmt.Printf(" binary: %s\n", p.binary)
|
||||||
|
fmt.Printf(" env file: %s (mode 0600)\n", p.envFile)
|
||||||
|
fmt.Printf(" data dir: %s\n", p.dataDir)
|
||||||
|
fmt.Printf(" console: http://%s:%s\n", values["WORKER_HOST"], values["WORKER_PORT"])
|
||||||
|
if opts.NoStart {
|
||||||
|
fmt.Printf(" status: enabled (not started, --no-start)\n")
|
||||||
|
} else {
|
||||||
|
fmt.Printf(" status: enabled and active\n")
|
||||||
|
}
|
||||||
|
if generatedPassword != "" {
|
||||||
|
fmt.Printf(" generated operator console password (login=%s): %s\n",
|
||||||
|
defaultLogin, generatedPassword)
|
||||||
|
fmt.Println(" This password is stored in the env file above; record it now.")
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// resolveInstallEnv applies the installer's precedence for every known
|
||||||
|
// worker variable: explicit flag > --env-file > process environment >
|
||||||
|
// built-in default. It fills defaults, enforces required values, and
|
||||||
|
// validates every resolved value is systemd-safe. Basic-auth XOR is
|
||||||
|
// enforced here; password generation happens in Install so the value
|
||||||
|
// can be printed.
|
||||||
|
func resolveInstallEnv(opts InstallOptions, name string, fileEnv map[string]string) (map[string]string, error) {
|
||||||
|
flagVals := map[string]string{
|
||||||
|
"RSMON_URL": opts.URL,
|
||||||
|
"RSMON_TOKEN": opts.Token,
|
||||||
|
"PUBLIC_URL": opts.PublicURL,
|
||||||
|
"WORKER_HOST": opts.Host,
|
||||||
|
"WORKER_PORT": opts.Port,
|
||||||
|
"WORKER_LOGIN": opts.Login,
|
||||||
|
"WORKER_PASSWORD": opts.Password,
|
||||||
|
}
|
||||||
|
values := make(map[string]string, len(installEnvKeys))
|
||||||
|
for _, key := range installEnvKeys {
|
||||||
|
if v, ok := flagVals[key]; ok && strings.TrimSpace(v) != "" {
|
||||||
|
values[key] = v
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if fileEnv != nil {
|
||||||
|
if v, ok := fileEnv[key]; ok && strings.TrimSpace(v) != "" {
|
||||||
|
values[key] = v
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if v := os.Getenv(key); strings.TrimSpace(v) != "" {
|
||||||
|
values[key] = v
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if values["RSMON_URL"] == "" {
|
||||||
|
values["RSMON_URL"] = DefaultURL
|
||||||
|
}
|
||||||
|
if values["WORKER_HOST"] == "" {
|
||||||
|
values["WORKER_HOST"] = "127.0.0.1"
|
||||||
|
}
|
||||||
|
if values["WORKER_PORT"] == "" {
|
||||||
|
if name == "" {
|
||||||
|
values["WORKER_PORT"] = defaultPort
|
||||||
|
} else {
|
||||||
|
return nil, fmt.Errorf(
|
||||||
|
"--name %q requires WORKER_PORT (the default %s belongs to the primary instance); "+
|
||||||
|
"set it via --port, WORKER_PORT, or the env file", name, defaultPort)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if strings.TrimSpace(values["RSMON_TOKEN"]) == "" {
|
||||||
|
return nil, errors.New("RSMON_TOKEN is required (set --token/--token-file, WORKER_TOKEN in --env-file, or RSMON_TOKEN in the environment/.env)")
|
||||||
|
}
|
||||||
|
if err := ValidateURL(values["RSMON_URL"]); err != nil {
|
||||||
|
return nil, fmt.Errorf("RSMON_URL: %w", err)
|
||||||
|
}
|
||||||
|
if err := ValidateToken(values["RSMON_TOKEN"]); err != nil {
|
||||||
|
return nil, fmt.Errorf("RSMON_TOKEN: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Basic auth: both set or both empty. Generation happens later.
|
||||||
|
login, pass := values["WORKER_LOGIN"], values["WORKER_PASSWORD"]
|
||||||
|
if (login == "") != (pass == "") {
|
||||||
|
return nil, fmt.Errorf("WORKER_LOGIN and WORKER_PASSWORD must both be set or both be empty")
|
||||||
|
}
|
||||||
|
|
||||||
|
// PUBLIC_URL is the canonical advertised origin. When both the
|
||||||
|
// canonical and the legacy WORKER_URL resolve, the legacy variable
|
||||||
|
// is superseded and must not be written to a fresh env file.
|
||||||
|
if values["PUBLIC_URL"] != "" {
|
||||||
|
delete(values, "WORKER_URL")
|
||||||
|
}
|
||||||
|
// Canonical PUBLIC_URL is held to the strict origin shape; the
|
||||||
|
// legacy WORKER_URL only to the tolerant absolute-URL check so env
|
||||||
|
// files that previously installed keep working.
|
||||||
|
if v := values["PUBLIC_URL"]; v != "" {
|
||||||
|
if err := distworker.ValidatePublicURL(v); err != nil {
|
||||||
|
return nil, fmt.Errorf("PUBLIC_URL: %w", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if v := values["WORKER_URL"]; v != "" {
|
||||||
|
if err := distworker.ValidateAdvertisedURL(v); err != nil {
|
||||||
|
return nil, fmt.Errorf("WORKER_URL: %w", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Validate every value we will write is systemd/docker safe.
|
||||||
|
for _, key := range installEnvKeys {
|
||||||
|
v, ok := values[key]
|
||||||
|
if !ok || v == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if err := validateEnvValue(key, v); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return values, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// renderEnvFile produces the canonical KEY=VALUE env file in a stable
|
||||||
|
// order. Empty values are omitted.
|
||||||
|
func renderEnvFile(values map[string]string) []byte {
|
||||||
|
var b strings.Builder
|
||||||
|
for _, key := range installEnvKeys {
|
||||||
|
v := values[key]
|
||||||
|
if v == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
fmt.Fprintf(&b, "%s=%s\n", key, v)
|
||||||
|
}
|
||||||
|
return []byte(b.String())
|
||||||
|
}
|
||||||
|
|
||||||
|
func generatePassword(nBytes int) (string, error) {
|
||||||
|
buf := make([]byte, nBytes)
|
||||||
|
if _, err := rand.Read(buf); err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
return hex.EncodeToString(buf), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// systemdUnitFor renders the hardened binary systemd unit for an
|
||||||
|
// instance. It mirrors the hardening the operator console expects
|
||||||
|
// (CAP_NET_RAW for ping, docker.service ordering for Compose
|
||||||
|
// discovery, a private writable data tree) while staying a plain
|
||||||
|
// Type=simple root service.
|
||||||
|
func systemdUnitFor(p paths) string {
|
||||||
|
description := "RSMon distributed monitoring worker"
|
||||||
|
if p.name != "" {
|
||||||
|
description += " (" + p.name + ")"
|
||||||
|
}
|
||||||
|
return fmt.Sprintf(`[Unit]
|
||||||
|
Description=%s
|
||||||
|
Documentation=https://rocketgit.ru/rsmon/worker
|
||||||
|
After=network-online.target docker.service
|
||||||
|
Wants=network-online.target
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
Type=simple
|
||||||
|
User=root
|
||||||
|
Group=root
|
||||||
|
Environment=HOME=%[2]s
|
||||||
|
Environment=RSMON_WEBAPP_DATA_DIR=%[2]s/webapp
|
||||||
|
EnvironmentFile=%[3]s
|
||||||
|
WorkingDirectory=%[2]s
|
||||||
|
ExecStart=%[4]s
|
||||||
|
Restart=always
|
||||||
|
RestartSec=5s
|
||||||
|
TimeoutStopSec=20s
|
||||||
|
AmbientCapabilities=CAP_NET_RAW
|
||||||
|
CapabilityBoundingSet=CAP_NET_RAW
|
||||||
|
NoNewPrivileges=true
|
||||||
|
PrivateTmp=true
|
||||||
|
ProtectHome=true
|
||||||
|
ProtectSystem=full
|
||||||
|
ReadWritePaths=%[2]s
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=multi-user.target
|
||||||
|
`, description, p.dataDir, p.envFile, p.binary)
|
||||||
|
}
|
||||||
|
|
||||||
|
// dockerUnitFor renders the Docker-backed systemd unit. The container
|
||||||
|
// always mounts its data volume at the in-image /var/lib/rsmon-worker;
|
||||||
|
// only the volume (and container) name vary by instance.
|
||||||
|
func dockerUnitFor(p paths, image string) string {
|
||||||
|
description := "RSMon distributed monitoring worker (Docker)"
|
||||||
|
if p.name != "" {
|
||||||
|
description += " (" + p.name + ")"
|
||||||
|
}
|
||||||
|
return fmt.Sprintf(`[Unit]
|
||||||
|
Description=%s
|
||||||
|
Documentation=https://rocketgit.ru/rsmon/worker
|
||||||
|
After=network-online.target docker.service
|
||||||
|
Requires=docker.service
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
Type=simple
|
||||||
|
User=root
|
||||||
|
ExecStartPre=-docker rm -f %[2]s
|
||||||
|
ExecStart=docker run --rm --name %[2]s --network host --cap-add NET_RAW --env-file %[3]s -v %[4]s:/var/lib/rsmon-worker %[5]s
|
||||||
|
ExecStop=docker stop %[2]s
|
||||||
|
Restart=on-failure
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=multi-user.target
|
||||||
|
`, description, p.container, p.envFile, p.volume, image)
|
||||||
|
}
|
||||||
|
|
||||||
|
func ValidateURL(raw string) error {
|
||||||
|
if strings.ContainsAny(raw, "\r\n") {
|
||||||
|
return fmt.Errorf("server URL must be an absolute HTTP(S) URL")
|
||||||
|
}
|
||||||
|
u, err := url.Parse(raw)
|
||||||
|
if err != nil || u.Host == "" || (u.Scheme != "https" && u.Scheme != "http") {
|
||||||
|
return fmt.Errorf("server URL must be an absolute HTTP(S) URL")
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Environment renders the minimal legacy env file (URL/token/host).
|
||||||
|
// Kept for callers and tests that depend on the classic shape; the
|
||||||
|
// installer now writes the full resolved set via renderEnvFile.
|
||||||
|
func Environment(serverURL, token string) []byte {
|
||||||
|
return []byte("RSMON_URL=" + serverURL + "\nRSMON_TOKEN=" + token + "\nWORKER_HOST=127.0.0.1\n")
|
||||||
|
}
|
||||||
|
|
||||||
|
func ValidateToken(token string) error {
|
||||||
|
if strings.TrimSpace(token) == "" || strings.ContainsAny(token, "\r\n") {
|
||||||
|
return errors.New("worker token must be non-empty and contain no newlines")
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// ValidateEnvironmentFile checks a worker credentials file before it
|
||||||
|
// is used for installation. It enforces strict, systemd-safe
|
||||||
|
// KEY=VALUE syntax (no quoting, interpolation, or whitespace in
|
||||||
|
// values) and requires RSMON_URL and RSMON_TOKEN.
|
||||||
|
func ValidateEnvironmentFile(path string) error {
|
||||||
|
data, err := os.ReadFile(path)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("read worker environment: %w", err)
|
||||||
|
}
|
||||||
|
_, err = parseEnvironmentContent(data)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// parseEnvironmentContent validates systemd-safe KEY=VALUE environment
|
||||||
|
// content and returns the parsed map. Values must not use quoting,
|
||||||
|
// interpolation, or whitespace; RSMON_URL and RSMON_TOKEN are required and
|
||||||
|
// must not be duplicated. Operating on the already-read bytes (rather
|
||||||
|
// than a path) keeps callers free of read-to-validate TOCTOU races.
|
||||||
|
func parseEnvironmentContent(data []byte) (map[string]string, error) {
|
||||||
|
values := make(map[string]string)
|
||||||
|
seenRequired := make(map[string]bool)
|
||||||
|
for number, line := range strings.Split(string(data), "\n") {
|
||||||
|
lineNumber := number + 1
|
||||||
|
if line == "" || strings.HasPrefix(line, "#") {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if strings.ContainsRune(line, '\r') {
|
||||||
|
return nil, fmt.Errorf("worker environment line %d contains a carriage return", lineNumber)
|
||||||
|
}
|
||||||
|
key, value, ok := strings.Cut(line, "=")
|
||||||
|
if !ok || !envKeyPattern.MatchString(key) {
|
||||||
|
return nil, fmt.Errorf("worker environment line %d must use KEY=VALUE syntax", lineNumber)
|
||||||
|
}
|
||||||
|
if err := validateEnvValue(key, value); err != nil {
|
||||||
|
return nil, fmt.Errorf("worker environment line %d: %w", lineNumber, err)
|
||||||
|
}
|
||||||
|
if key == "RSMON_URL" || key == "RSMON_TOKEN" {
|
||||||
|
if seenRequired[key] {
|
||||||
|
return nil, fmt.Errorf("worker environment line %d duplicates %s", lineNumber, key)
|
||||||
|
}
|
||||||
|
seenRequired[key] = true
|
||||||
|
}
|
||||||
|
values[key] = value
|
||||||
|
}
|
||||||
|
if err := ValidateURL(values["RSMON_URL"]); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if err := ValidateToken(values["RSMON_TOKEN"]); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return values, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// validateEnvValue enforces the value rules shared by the strict env
|
||||||
|
// file parser and the installer's resolved values: no whitespace, no
|
||||||
|
// shell quoting, and no interpolation metacharacters. This keeps the
|
||||||
|
// file unambiguous across systemd EnvironmentFile and docker --env-file.
|
||||||
|
func validateEnvValue(key, value string) error {
|
||||||
|
if strings.ContainsAny(value, "\r\n") {
|
||||||
|
return fmt.Errorf("%s contains a newline", key)
|
||||||
|
}
|
||||||
|
if strings.IndexFunc(value, unicode.IsSpace) >= 0 || strings.ContainsAny(value, "$\\\"'") {
|
||||||
|
return fmt.Errorf("%s uses unsupported quoting, interpolation, or whitespace", key)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func ValidateImage(image string) error {
|
||||||
|
if !dockerImagePattern.MatchString(image) {
|
||||||
|
return errors.New("Docker image must be an immutable repository@sha256:<64 lowercase hex characters> reference")
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// DockerUnit renders the Docker-backed systemd unit for the classic
|
||||||
|
// single instance. Kept for compatibility; named installs go through
|
||||||
|
// dockerUnitFor.
|
||||||
|
func DockerUnit(image string) string {
|
||||||
|
return dockerUnitFor(resolvePaths(""), image)
|
||||||
|
}
|
||||||
|
|
||||||
|
func copyAtomic(src, dst string, mode os.FileMode) error {
|
||||||
|
in, err := os.Open(src)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer in.Close()
|
||||||
|
return atomicFile(dst, mode, func(out *os.File) error {
|
||||||
|
_, err := io.Copy(out, in)
|
||||||
|
return err
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func writeAtomic(dst string, data []byte, mode os.FileMode) error {
|
||||||
|
return atomicFile(dst, mode, func(out *os.File) error {
|
||||||
|
_, err := out.Write(data)
|
||||||
|
return err
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func atomicFile(dst string, mode os.FileMode, write func(*os.File) error) error {
|
||||||
|
if err := os.MkdirAll(filepath.Dir(dst), 0755); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
tmp, err := os.CreateTemp(filepath.Dir(dst), ".rsmon-worker-*")
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
name := tmp.Name()
|
||||||
|
defer os.Remove(name)
|
||||||
|
if err := tmp.Chmod(mode); err != nil {
|
||||||
|
tmp.Close()
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := write(tmp); err != nil {
|
||||||
|
tmp.Close()
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := tmp.Sync(); err != nil {
|
||||||
|
tmp.Close()
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := tmp.Close(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return os.Rename(name, dst)
|
||||||
|
}
|
||||||
|
|
||||||
|
func command(name string, args ...string) error {
|
||||||
|
cmd := exec.Command(name, args...)
|
||||||
|
cmd.Stdout = os.Stdout
|
||||||
|
cmd.Stderr = os.Stderr
|
||||||
|
if err := cmd.Run(); err != nil {
|
||||||
|
return fmt.Errorf("%s failed: %w", name, err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
409
internal/installer/install_test.go
Обычный файл
409
internal/installer/install_test.go
Обычный файл
@@ -0,0 +1,409 @@
|
|||||||
|
package installer
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestValidateURL(t *testing.T) {
|
||||||
|
for _, raw := range []string{"https://rsmon.ru", "http://localhost:7401"} {
|
||||||
|
if err := ValidateURL(raw); err != nil {
|
||||||
|
t.Fatalf("ValidateURL(%q): %v", raw, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, raw := range []string{"", "rsmon.ru", "file:///tmp/x"} {
|
||||||
|
if err := ValidateURL(raw); err == nil {
|
||||||
|
t.Fatalf("ValidateURL(%q) succeeded", raw)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestValidateToken(t *testing.T) {
|
||||||
|
if err := ValidateToken("token"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, token := range []string{"", " ", "token\nRSMON_URL=https://evil.test"} {
|
||||||
|
if err := ValidateToken(token); err == nil {
|
||||||
|
t.Fatalf("ValidateToken(%q) succeeded", token)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestValidateEnvironmentFile(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
contents string
|
||||||
|
valid bool
|
||||||
|
}{
|
||||||
|
{name: "valid", contents: "# Worker credentials\nRSMON_URL=https://rsmon.ru\nRSMON_TOKEN=secret\n\n", valid: true},
|
||||||
|
{name: "missing URL", contents: "RSMON_TOKEN=secret\n"},
|
||||||
|
{name: "missing token", contents: "RSMON_URL=https://rsmon.ru\n"},
|
||||||
|
{name: "invalid URL", contents: "RSMON_URL=file:///tmp/worker\nRSMON_TOKEN=secret\n"},
|
||||||
|
{name: "additional settings", contents: "RSMON_URL=https://rsmon.ru\nRSMON_TOKEN=secret\nWORKER_HOST=0.0.0.0\nWORKER_URL=\n", valid: true},
|
||||||
|
{name: "dotenv interpolation", contents: "RSMON_URL=https://rsmon.ru\nRSMON_TOKEN=${TOKEN}\n"},
|
||||||
|
{name: "dotenv export", contents: "export RSMON_URL=https://rsmon.ru\nRSMON_TOKEN=secret\n"},
|
||||||
|
{name: "YAML assignment", contents: "RSMON_URL: https://rsmon.ru\nRSMON_TOKEN=secret\n"},
|
||||||
|
{name: "malformed key", contents: "RSMON-URL=https://rsmon.ru\nRSMON_TOKEN=secret\n"},
|
||||||
|
{name: "missing assignment", contents: "RSMON_URL=https://rsmon.ru\nRSMON_TOKEN\n"},
|
||||||
|
{name: "quoted value", contents: "RSMON_URL=https://rsmon.ru\nRSMON_TOKEN=\"secret\"\n"},
|
||||||
|
{name: "whitespace", contents: "RSMON_URL=https://rsmon.ru\nRSMON_TOKEN=secret value\n"},
|
||||||
|
{name: "duplicate URL", contents: "RSMON_URL=https://rsmon.ru\nRSMON_URL=https://evil.test\nRSMON_TOKEN=secret\n"},
|
||||||
|
{name: "duplicate empty token", contents: "RSMON_URL=https://rsmon.ru\nRSMON_TOKEN=\nRSMON_TOKEN=secret\n"},
|
||||||
|
{name: "carriage return", contents: "RSMON_URL=https://rsmon.ru\r\nRSMON_TOKEN=secret\r\n"},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
path := filepath.Join(t.TempDir(), "worker.env")
|
||||||
|
if err := os.WriteFile(path, []byte(tt.contents), 0600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
err := ValidateEnvironmentFile(path)
|
||||||
|
if tt.valid && err != nil {
|
||||||
|
t.Fatalf("ValidateEnvironmentFile() error = %v", err)
|
||||||
|
}
|
||||||
|
if !tt.valid && err == nil {
|
||||||
|
t.Fatal("ValidateEnvironmentFile() succeeded")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestValidateEnvironmentFileInputErrors(t *testing.T) {
|
||||||
|
if err := ValidateEnvironmentFile(filepath.Join(t.TempDir(), "missing")); err == nil {
|
||||||
|
t.Fatal("missing environment file accepted")
|
||||||
|
}
|
||||||
|
if err := ValidateEnvironmentFile(t.TempDir()); err == nil {
|
||||||
|
t.Fatal("directory accepted as an environment file")
|
||||||
|
}
|
||||||
|
if os.Geteuid() == 0 {
|
||||||
|
t.Skip("root can read mode-000 files")
|
||||||
|
}
|
||||||
|
path := filepath.Join(t.TempDir(), "unreadable")
|
||||||
|
if err := os.WriteFile(path, []byte("RSMON_URL=https://rsmon.ru\nRSMON_TOKEN=secret\n"), 0000); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := ValidateEnvironmentFile(path); err == nil {
|
||||||
|
t.Fatal("unreadable environment file accepted")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestResolveInstallEnvPublicURLWins verifies the installer canonicalizes
|
||||||
|
// the advertised origin: PUBLIC_URL is written and the legacy WORKER_URL
|
||||||
|
// is dropped from the resolved env when both are present.
|
||||||
|
func TestResolveInstallEnvPublicURLWins(t *testing.T) {
|
||||||
|
v, err := resolveInstallEnv(InstallOptions{}, "", map[string]string{
|
||||||
|
"RSMON_URL": "https://rsmon.ru",
|
||||||
|
"RSMON_TOKEN": "secret",
|
||||||
|
"PUBLIC_URL": "https://worker.example.com",
|
||||||
|
"WORKER_URL": "http://legacy.example.com",
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if v["PUBLIC_URL"] != "https://worker.example.com" {
|
||||||
|
t.Fatalf("PUBLIC_URL not resolved: %+v", v)
|
||||||
|
}
|
||||||
|
if _, ok := v["WORKER_URL"]; ok {
|
||||||
|
t.Fatalf("legacy WORKER_URL must be dropped when PUBLIC_URL is set: %+v", v)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestResolveInstallEnvLegacyWorkerURLPassesThrough keeps the bounded
|
||||||
|
// migration: an env file that only carries the legacy WORKER_URL still
|
||||||
|
// resolves and is written unchanged so existing installs upgrade in place.
|
||||||
|
func TestResolveInstallEnvLegacyWorkerURLPassesThrough(t *testing.T) {
|
||||||
|
v, err := resolveInstallEnv(InstallOptions{}, "", map[string]string{
|
||||||
|
"RSMON_URL": "https://rsmon.ru",
|
||||||
|
"RSMON_TOKEN": "secret",
|
||||||
|
"WORKER_URL": "https://legacy.example.com",
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if v["WORKER_URL"] != "https://legacy.example.com" {
|
||||||
|
t.Fatalf("legacy WORKER_URL not preserved: %+v", v)
|
||||||
|
}
|
||||||
|
if v["PUBLIC_URL"] != "" {
|
||||||
|
t.Fatalf("PUBLIC_URL must stay empty: %+v", v)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestResolveInstallEnvLegacyWorkerURLTolerant verifies the bounded
|
||||||
|
// migration does not newly reject legacy shapes that previously
|
||||||
|
// installed (a path-bearing WORKER_URL) while a path-bearing PUBLIC_URL
|
||||||
|
// stays strict.
|
||||||
|
func TestResolveInstallEnvLegacyWorkerURLTolerant(t *testing.T) {
|
||||||
|
v, err := resolveInstallEnv(InstallOptions{}, "", map[string]string{
|
||||||
|
"RSMON_URL": "https://rsmon.ru",
|
||||||
|
"RSMON_TOKEN": "secret",
|
||||||
|
"WORKER_URL": "https://legacy.example.com/web",
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("legacy WORKER_URL with a path must keep installing: %v", err)
|
||||||
|
}
|
||||||
|
if v["WORKER_URL"] != "https://legacy.example.com/web" {
|
||||||
|
t.Fatalf("legacy WORKER_URL not preserved: %+v", v)
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err = resolveInstallEnv(InstallOptions{}, "", map[string]string{
|
||||||
|
"RSMON_URL": "https://rsmon.ru",
|
||||||
|
"RSMON_TOKEN": "secret",
|
||||||
|
"PUBLIC_URL": "https://worker.example.com/web",
|
||||||
|
})
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("path-bearing canonical PUBLIC_URL must be rejected")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestResolveInstallEnvPublicURLFlagBeatsEnv verifies the --public-url
|
||||||
|
// flag follows the installer precedence: the flag wins over the env file
|
||||||
|
// and the legacy WORKER_URL is dropped when PUBLIC_URL is present.
|
||||||
|
func TestResolveInstallEnvPublicURLFlagBeatsEnv(t *testing.T) {
|
||||||
|
v, err := resolveInstallEnv(InstallOptions{PublicURL: "https://flag.example.com"}, "", map[string]string{
|
||||||
|
"RSMON_URL": "https://rsmon.ru",
|
||||||
|
"RSMON_TOKEN": "secret",
|
||||||
|
"PUBLIC_URL": "https://file.example.com",
|
||||||
|
"WORKER_URL": "https://legacy.example.com",
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if v["PUBLIC_URL"] != "https://flag.example.com" {
|
||||||
|
t.Fatalf("--public-url flag must win: %+v", v)
|
||||||
|
}
|
||||||
|
if _, ok := v["WORKER_URL"]; ok {
|
||||||
|
t.Fatalf("legacy WORKER_URL must be dropped when PUBLIC_URL is set: %+v", v)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestResolveInstallEnvRejectsMalformedPublicURL verifies the installer
|
||||||
|
// rejects an advertised origin that violates the plan's origin shape
|
||||||
|
// (path, userinfo, and non-http(s) schemes).
|
||||||
|
func TestResolveInstallEnvRejectsMalformedPublicURL(t *testing.T) {
|
||||||
|
for _, bad := range []string{
|
||||||
|
"https://worker.example.com/web",
|
||||||
|
"https://user:pass@worker.example.com",
|
||||||
|
"ftp://worker.example.com",
|
||||||
|
"worker.example.com",
|
||||||
|
} {
|
||||||
|
t.Run(bad, func(t *testing.T) {
|
||||||
|
_, err := resolveInstallEnv(InstallOptions{}, "", map[string]string{
|
||||||
|
"RSMON_URL": "https://rsmon.ru",
|
||||||
|
"RSMON_TOKEN": "secret",
|
||||||
|
"PUBLIC_URL": bad,
|
||||||
|
})
|
||||||
|
if err == nil {
|
||||||
|
t.Fatalf("PUBLIC_URL=%q accepted", bad)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestRenderEnvFileOrder includes the canonical PUBLIC_URL ordering.
|
||||||
|
func TestRenderEnvFilePublicURLEmptyOmitted(t *testing.T) {
|
||||||
|
got := string(renderEnvFile(map[string]string{
|
||||||
|
"RSMON_URL": "https://rsmon.ru",
|
||||||
|
"RSMON_TOKEN": "secret",
|
||||||
|
"PUBLIC_URL": "",
|
||||||
|
"WORKER_URL": "",
|
||||||
|
}))
|
||||||
|
if strings.Contains(got, "PUBLIC_URL=") || strings.Contains(got, "WORKER_URL=") {
|
||||||
|
t.Fatalf("empty public URL keys must be omitted: %q", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestEnvironment(t *testing.T) {
|
||||||
|
got := string(Environment("https://example.test", "secret"))
|
||||||
|
for _, want := range []string{"RSMON_URL=https://example.test\n", "RSMON_TOKEN=secret\n", "WORKER_HOST=127.0.0.1\n"} {
|
||||||
|
if !strings.Contains(got, want) {
|
||||||
|
t.Fatalf("environment missing %q", want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestShellQuote(t *testing.T) {
|
||||||
|
if got, want := shellQuote("a'b"), `'a'\''b'`; got != want {
|
||||||
|
t.Fatalf("shellQuote() = %q, want %q", got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestValidateImage(t *testing.T) {
|
||||||
|
const image = "reg.rsxx.ru/rsmon/rsmon-worker@sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"
|
||||||
|
if err := ValidateImage(image); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, image := range []string{"", "-bad", "image name", "image%stest", `image"test`, "reg.rsxx.ru/rsmon/rsmon-worker:latest", "reg.rsxx.ru/rsmon/rsmon-worker@sha256:short", "reg.rsxx.ru/rsmon/rsmon-worker@sha256:0123456789ABCDEF0123456789abcdef0123456789abcdef0123456789abcdef"} {
|
||||||
|
if err := ValidateImage(image); err == nil {
|
||||||
|
t.Fatalf("ValidateImage(%q) succeeded", image)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSystemdUnits(t *testing.T) {
|
||||||
|
primary := systemdUnitFor(resolvePaths(""))
|
||||||
|
if !strings.Contains(primary, "Type=simple\nUser=root\n") ||
|
||||||
|
!strings.Contains(primary, "ExecStart=/usr/local/bin/rsmon-worker\n") ||
|
||||||
|
!strings.Contains(primary, "EnvironmentFile=/etc/rsmon-worker/worker.env\n") {
|
||||||
|
t.Fatal("binary systemd unit is not the simple root service at the classic paths")
|
||||||
|
}
|
||||||
|
if !strings.Contains(primary, "After=network-online.target docker.service") ||
|
||||||
|
!strings.Contains(primary, "CAP_NET_RAW") || !strings.Contains(primary, "ProtectSystem=full") {
|
||||||
|
t.Fatal("binary systemd unit must order after docker and harden for ping/compose")
|
||||||
|
}
|
||||||
|
|
||||||
|
named := systemdUnitFor(resolvePaths("edge"))
|
||||||
|
for _, want := range []string{
|
||||||
|
"ExecStart=/usr/local/bin/rsmon-worker-edge",
|
||||||
|
"EnvironmentFile=/etc/rsmon-worker-edge/worker.env",
|
||||||
|
"RSMON_WEBAPP_DATA_DIR=/var/lib/rsmon-worker-edge/webapp",
|
||||||
|
"(edge)",
|
||||||
|
} {
|
||||||
|
if !strings.Contains(named, want) {
|
||||||
|
t.Fatalf("named binary unit missing %q", want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const image = "reg.rsxx.ru/rsmon/rsmon-worker@sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"
|
||||||
|
unit := DockerUnit(image)
|
||||||
|
for _, want := range []string{"ExecStartPre=-docker rm -f rsmon-worker", "docker run --rm", image} {
|
||||||
|
if !strings.Contains(unit, want) {
|
||||||
|
t.Fatalf("Docker systemd unit missing %q", want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
namedDocker := dockerUnitFor(resolvePaths("edge"), image)
|
||||||
|
for _, want := range []string{"docker rm -f rsmon-worker-edge", "--name rsmon-worker-edge ", "-v rsmon-worker-data-edge:/var/lib/rsmon-worker"} {
|
||||||
|
if !strings.Contains(namedDocker, want) {
|
||||||
|
t.Fatalf("named Docker unit missing %q", want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestResolvePaths(t *testing.T) {
|
||||||
|
primary := resolvePaths("")
|
||||||
|
if primary.binary != "/usr/local/bin/rsmon-worker" ||
|
||||||
|
primary.configDir != "/etc/rsmon-worker" ||
|
||||||
|
primary.envFile != "/etc/rsmon-worker/worker.env" ||
|
||||||
|
primary.dataDir != "/var/lib/rsmon-worker" ||
|
||||||
|
primary.unitName != "rsmon-worker.service" ||
|
||||||
|
primary.unitFile != "/etc/systemd/system/rsmon-worker.service" ||
|
||||||
|
primary.container != "rsmon-worker" || primary.volume != "rsmon-worker-data" {
|
||||||
|
t.Fatalf("primary paths wrong: %+v", primary)
|
||||||
|
}
|
||||||
|
edge := resolvePaths("edge")
|
||||||
|
if edge.binary != "/usr/local/bin/rsmon-worker-edge" ||
|
||||||
|
edge.configDir != "/etc/rsmon-worker-edge" ||
|
||||||
|
edge.envFile != "/etc/rsmon-worker-edge/worker.env" ||
|
||||||
|
edge.dataDir != "/var/lib/rsmon-worker-edge" ||
|
||||||
|
edge.unitName != "rsmon-worker-edge.service" ||
|
||||||
|
edge.unitFile != "/etc/systemd/system/rsmon-worker-edge.service" ||
|
||||||
|
edge.container != "rsmon-worker-edge" || edge.volume != "rsmon-worker-data-edge" {
|
||||||
|
t.Fatalf("named paths wrong: %+v", edge)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestValidateInstanceName(t *testing.T) {
|
||||||
|
for _, n := range []string{"", "dev", "edge-1", "a", "ab"} {
|
||||||
|
if err := validateInstanceName(n); err != nil {
|
||||||
|
t.Fatalf("validateInstanceName(%q): %v", n, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, n := range []string{"Dev", "dev_", "-dev", "dev-", "a.b", strings.Repeat("a", 33), "dev zone"} {
|
||||||
|
if err := validateInstanceName(n); err == nil {
|
||||||
|
t.Fatalf("validateInstanceName(%q) succeeded", n)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestResolveInstallEnv(t *testing.T) {
|
||||||
|
// Force a deterministic process environment so precedence is exact.
|
||||||
|
t.Setenv("RSMON_URL", "https://proc.test")
|
||||||
|
t.Setenv("RSMON_TOKEN", "proc-token")
|
||||||
|
t.Setenv("WORKER_PORT", "29999")
|
||||||
|
t.Setenv("WORKER_LOGIN", "")
|
||||||
|
t.Setenv("WORKER_PASSWORD", "")
|
||||||
|
|
||||||
|
t.Run("flag beats process env", func(t *testing.T) {
|
||||||
|
v, err := resolveInstallEnv(InstallOptions{
|
||||||
|
URL: "https://flag.test", Token: "flag-token", Port: "28080",
|
||||||
|
}, "", nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if v["RSMON_URL"] != "https://flag.test" || v["RSMON_TOKEN"] != "flag-token" || v["WORKER_PORT"] != "28080" {
|
||||||
|
t.Fatalf("flag did not win: %+v", v)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("process env fills when flags empty", func(t *testing.T) {
|
||||||
|
v, err := resolveInstallEnv(InstallOptions{}, "", nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if v["RSMON_URL"] != "https://proc.test" || v["RSMON_TOKEN"] != "proc-token" || v["WORKER_PORT"] != "29999" {
|
||||||
|
t.Fatalf("process env not used: %+v", v)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("env file beats process env", func(t *testing.T) {
|
||||||
|
v, err := resolveInstallEnv(InstallOptions{}, "", map[string]string{
|
||||||
|
"RSMON_URL": "https://file.test", "RSMON_TOKEN": "file-token",
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if v["RSMON_URL"] != "https://file.test" || v["RSMON_TOKEN"] != "file-token" {
|
||||||
|
t.Fatalf("env file did not beat process: %+v", v)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("token required", func(t *testing.T) {
|
||||||
|
t.Setenv("RSMON_TOKEN", "")
|
||||||
|
if _, err := resolveInstallEnv(InstallOptions{}, "", nil); err == nil {
|
||||||
|
t.Fatal("missing token accepted")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("named instance requires port", func(t *testing.T) {
|
||||||
|
t.Setenv("WORKER_PORT", "")
|
||||||
|
if _, err := resolveInstallEnv(InstallOptions{}, "edge", nil); err == nil {
|
||||||
|
t.Fatal("named instance without port accepted")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("basic auth XOR rejected", func(t *testing.T) {
|
||||||
|
_, err := resolveInstallEnv(InstallOptions{Login: "admin"}, "", nil)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("login-only accepted")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRenderEnvFileOrderAndOmission(t *testing.T) {
|
||||||
|
got := string(renderEnvFile(map[string]string{
|
||||||
|
"WORKER_PORT": "27402",
|
||||||
|
"RSMON_TOKEN": "secret",
|
||||||
|
"RSMON_URL": "https://rsmon.ru",
|
||||||
|
"WORKER_LOGIN": "", // omitted
|
||||||
|
"WORKER_PASSWORD": "",
|
||||||
|
}))
|
||||||
|
want := "RSMON_URL=https://rsmon.ru\nRSMON_TOKEN=secret\nWORKER_PORT=27402\n"
|
||||||
|
if got != want {
|
||||||
|
t.Fatalf("renderEnvFile = %q, want %q", got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestValidateEnvValue(t *testing.T) {
|
||||||
|
for _, v := range []string{"secret", "abc-123", "https://rsmon.ru", "27402"} {
|
||||||
|
if err := validateEnvValue("KEY", v); err != nil {
|
||||||
|
t.Fatalf("validateEnvValue(%q): %v", v, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, v := range []string{"a b", `"q"`, `'q'`, "a$b", `a\b`, "a\nb"} {
|
||||||
|
if err := validateEnvValue("KEY", v); err == nil {
|
||||||
|
t.Fatalf("validateEnvValue(%q) succeeded", v)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
769
internal/installer/sourceactivate.go
Обычный файл
769
internal/installer/sourceactivate.go
Обычный файл
@@ -0,0 +1,769 @@
|
|||||||
|
package installer
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"rocketgit.ru/rsmon/worker/internal/sshinstall"
|
||||||
|
)
|
||||||
|
|
||||||
|
// ActivationOptions drives work package 4 of docs/source-installation.md:
|
||||||
|
// the atomic activation of the staged worker binary. It reuses the
|
||||||
|
// classic installer's environment resolution (resolveInstallEnv /
|
||||||
|
// renderEnvFile), on-disk layout (resolvePaths), and hardened systemd
|
||||||
|
// unit renderer, then installs the binary, env file, data dir, and the
|
||||||
|
// detected init's service definition atomically, starts/restarts the
|
||||||
|
// worker, and verifies the process and /healthz before declaring the
|
||||||
|
// activation a success. Any activation/start/health failure rolls back to
|
||||||
|
// the previous working install.
|
||||||
|
type ActivationOptions struct {
|
||||||
|
// Activate performs the atomic install + service activation after the
|
||||||
|
// staging build. When false SourceInstall stops at the staging build
|
||||||
|
// (the work-package-3 boundary) and touches no service configuration.
|
||||||
|
Activate bool
|
||||||
|
|
||||||
|
// Name is the instance name ("" = the primary rsmon-worker service).
|
||||||
|
// A named instance gets rsmon-worker-<name> paths and its own unit,
|
||||||
|
// matching the classic installer's multi-instance layout.
|
||||||
|
Name string
|
||||||
|
// URL is RSMON_URL (default https://rsmon.ru).
|
||||||
|
URL string
|
||||||
|
// Token is the worker API token (RSMON_TOKEN). Required for
|
||||||
|
// activation; prefer supplying it through a file at the CLI layer so
|
||||||
|
// it never appears in the process list.
|
||||||
|
Token string
|
||||||
|
// PublicURL is the advertised public origin (PUBLIC_URL).
|
||||||
|
PublicURL string
|
||||||
|
// Host is the worker webapp bind address (WORKER_HOST; default
|
||||||
|
// 127.0.0.1 as resolved by the classic installer).
|
||||||
|
Host string
|
||||||
|
// Port is the worker webapp bind port (WORKER_PORT; default 27401,
|
||||||
|
// required for named instances).
|
||||||
|
Port string
|
||||||
|
// Login / Password are the operator-console basic-auth credentials
|
||||||
|
// (WORKER_LOGIN / WORKER_PASSWORD). Both must be set or both empty.
|
||||||
|
Login string
|
||||||
|
Password string
|
||||||
|
// EnvFile is a local systemd-safe env file uploaded instead of the
|
||||||
|
// individual knobs. Validated with the classic strict parser.
|
||||||
|
EnvFile string
|
||||||
|
// NoStart installs the binary, env, data dir, and service definition
|
||||||
|
// without starting or restarting the worker.
|
||||||
|
NoStart bool
|
||||||
|
}
|
||||||
|
|
||||||
|
// ActivationResult records where and how the staged worker was activated.
|
||||||
|
type ActivationResult struct {
|
||||||
|
Binary string // installed binary path
|
||||||
|
ConfigDir string
|
||||||
|
EnvFile string
|
||||||
|
DataDir string
|
||||||
|
UnitFile string // empty when no service definition was installed
|
||||||
|
UnitName string // systemd unit / rc-service name
|
||||||
|
Supervisor string // "systemd", "openrc", or "supervisor" (embedded fallback)
|
||||||
|
Started bool
|
||||||
|
}
|
||||||
|
|
||||||
|
// installOptions projects the activation knobs onto the classic
|
||||||
|
// installer's option shape so the shared env resolution is reused.
|
||||||
|
func (a ActivationOptions) installOptions() InstallOptions {
|
||||||
|
return InstallOptions{
|
||||||
|
URL: a.URL,
|
||||||
|
Token: a.Token,
|
||||||
|
PublicURL: a.PublicURL,
|
||||||
|
Host: a.Host,
|
||||||
|
Port: a.Port,
|
||||||
|
Login: a.Login,
|
||||||
|
Password: a.Password,
|
||||||
|
Name: strings.TrimSpace(a.Name),
|
||||||
|
NoStart: a.NoStart,
|
||||||
|
EnvFile: a.EnvFile,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// renderEnv resolves the full worker environment exactly as the classic
|
||||||
|
// installer would and renders the canonical systemd-safe env file. The
|
||||||
|
// env file (when configured) is read exactly once and validated from the
|
||||||
|
// in-memory bytes, so a hostile local writer cannot swap the file
|
||||||
|
// between the read used for validation and the read used for the render.
|
||||||
|
func (a ActivationOptions) renderEnv() ([]byte, error) {
|
||||||
|
name := strings.TrimSpace(a.Name)
|
||||||
|
if err := validateInstanceName(name); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
var fileEnv map[string]string
|
||||||
|
if a.EnvFile != "" {
|
||||||
|
data, err := os.ReadFile(a.EnvFile)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("--env-file: %w", err)
|
||||||
|
}
|
||||||
|
fe, perr := parseEnvironmentContent(data)
|
||||||
|
if perr != nil {
|
||||||
|
return nil, fmt.Errorf("--env-file: %w", perr)
|
||||||
|
}
|
||||||
|
fileEnv = fe
|
||||||
|
}
|
||||||
|
values, err := resolveInstallEnv(a.installOptions(), name, fileEnv)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return renderEnvFile(values), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// unitContent renders the service definition for the detected init
|
||||||
|
// system. systemd gets the classic hardened unit; OpenRC gets a native
|
||||||
|
// init script. An unknown init produces no unit (the no-service gate:
|
||||||
|
// the embedded supervisor still installs and manages the worker, but no
|
||||||
|
// native service definition is written).
|
||||||
|
func unitContent(init sshinstall.InitSystem, p paths) (content string, file string, mode string) {
|
||||||
|
svcName := strings.TrimSuffix(p.unitName, ".service")
|
||||||
|
switch init {
|
||||||
|
case sshinstall.InitSystemd:
|
||||||
|
return systemdUnitFor(p), p.unitFile, "0644"
|
||||||
|
case sshinstall.InitOpenRC:
|
||||||
|
return openrcInitFor(p), "/etc/init.d/" + svcName, "0755"
|
||||||
|
default:
|
||||||
|
return "", "", ""
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// runningInitScript reports the init system that is *actually running*
|
||||||
|
// (not merely installed): systemd, openrc, or none. The installer writes
|
||||||
|
// the service definition for the detected init but drives start/restart
|
||||||
|
// through whichever supervisor is usable right now; a container or chroot
|
||||||
|
// where no init runs falls back to the embedded supervisor.
|
||||||
|
const runningInitScript = `set -eu
|
||||||
|
if [ -d /run/systemd/system ] && command -v systemctl >/dev/null 2>&1; then
|
||||||
|
echo systemd
|
||||||
|
elif [ -e /run/openrc/softlevel ] && command -v rc-service >/dev/null 2>&1; then
|
||||||
|
echo openrc
|
||||||
|
else
|
||||||
|
echo none
|
||||||
|
fi`
|
||||||
|
|
||||||
|
// normalizeSupervisor bounds the running-init probe output to the three
|
||||||
|
// supervisor kinds the activation script understands.
|
||||||
|
func normalizeSupervisor(raw string) string {
|
||||||
|
switch strings.TrimSpace(raw) {
|
||||||
|
case "systemd", "openrc":
|
||||||
|
return strings.TrimSpace(raw)
|
||||||
|
default:
|
||||||
|
return "none"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// activateScript renders the single idempotent remote activation script.
|
||||||
|
// It is structured as a transaction over the previous install:
|
||||||
|
//
|
||||||
|
// 1. acquire an activation lock so concurrent runs cannot mutate the same
|
||||||
|
// deployment, and recover any interrupted prior activation from its
|
||||||
|
// leftover backup marker;
|
||||||
|
// 2. validate the staged binary (fails before any state is touched);
|
||||||
|
// 3. snapshot the prior binary/env/unit (and the unit's enable state)
|
||||||
|
// into a backup dir and write a recovery marker;
|
||||||
|
// 4. arm a rollback EXIT/HUP/INT/TERM trap that restores the snapshot,
|
||||||
|
// restores the prior enable state (or disables a freshly-installed
|
||||||
|
// unit on a fresh failure), and restarts the prior service;
|
||||||
|
// 5. install the new binary, env (mode 0600), data dir, and service
|
||||||
|
// definition atomically (temp file + rename);
|
||||||
|
// 6. start/restart through the active supervisor and verify the service
|
||||||
|
// stays active (systemd `is-active` / rc-service `status` / the
|
||||||
|
// embedded supervisor's zombie-aware pid check) and answers /healthz;
|
||||||
|
// 7. on success, drop the backup and release the lock; on failure the
|
||||||
|
// trap restores the prior install and the script exits non-zero.
|
||||||
|
//
|
||||||
|
// Reruns are idempotent: restart always stops the previous instance
|
||||||
|
// first, so exactly one worker process exists, and every temp file
|
||||||
|
// (backup dir, .new files, uploaded env/unit, lock) is removed on success
|
||||||
|
// and failure.
|
||||||
|
func activateScript(p activateParams) string {
|
||||||
|
r := strings.NewReplacer(
|
||||||
|
"@@STAGE@@", shellQuote(p.Stage),
|
||||||
|
"@@BINARY@@", shellQuote(p.Binary),
|
||||||
|
"@@CONFIG_DIR@@", shellQuote(p.ConfigDir),
|
||||||
|
"@@ENV_FILE@@", shellQuote(p.EnvFile),
|
||||||
|
"@@ENV_TMP@@", shellQuote(p.EnvTmp),
|
||||||
|
"@@DATA_DIR@@", shellQuote(p.DataDir),
|
||||||
|
"@@UNIT_TMP@@", shellQuote(p.UnitTmp),
|
||||||
|
"@@UNIT_FILE@@", shellQuote(p.UnitFile),
|
||||||
|
"@@UNIT_MODE@@", p.UnitMode,
|
||||||
|
"@@UNIT_NAME@@", shellQuote(p.UnitName),
|
||||||
|
"@@RC_NAME@@", shellQuote(p.RCName),
|
||||||
|
"@@SUPERVISOR@@", p.Supervisor,
|
||||||
|
"@@NO_START@@", p.NoStart,
|
||||||
|
)
|
||||||
|
return r.Replace(activateTemplate)
|
||||||
|
}
|
||||||
|
|
||||||
|
// activateTemplate is the remote script body. Sentinels (@@..@@) are
|
||||||
|
// substituted by activateScript; every operator-controlled value is
|
||||||
|
// single-quoted and validated on the controller side first.
|
||||||
|
const activateTemplate = `set -eu
|
||||||
|
|
||||||
|
stage=@@STAGE@@
|
||||||
|
binary=@@BINARY@@
|
||||||
|
config_dir=@@CONFIG_DIR@@
|
||||||
|
env_file=@@ENV_FILE@@
|
||||||
|
env_tmp=@@ENV_TMP@@
|
||||||
|
data_dir=@@DATA_DIR@@
|
||||||
|
unit_tmp=@@UNIT_TMP@@
|
||||||
|
unit_file=@@UNIT_FILE@@
|
||||||
|
unit_mode=@@UNIT_MODE@@
|
||||||
|
unit_name=@@UNIT_NAME@@
|
||||||
|
rc_name=@@RC_NAME@@
|
||||||
|
supervisor=@@SUPERVISOR@@
|
||||||
|
no_start=@@NO_START@@
|
||||||
|
|
||||||
|
backup_dir="$data_dir/.rsmon-backup"
|
||||||
|
lock_dir="$data_dir/.rsmon-activate.lock"
|
||||||
|
pid_file="$data_dir/worker.pid"
|
||||||
|
log_file="$data_dir/worker.log"
|
||||||
|
|
||||||
|
run_env() {
|
||||||
|
# Execute "$@" with a fresh environment built only from the env file
|
||||||
|
# plus PATH, HOME, and the webapp data dir. Sourcing the env file in a
|
||||||
|
# clean env -i shell prevents variables from a previous activation
|
||||||
|
# (for example WORKER_CLUSTER_ENABLED) from leaking into the new
|
||||||
|
# worker process after a rollback restores an older env file.
|
||||||
|
env -i PATH="/usr/bin:/bin:/sbin:/usr/sbin" HOME="$data_dir" \
|
||||||
|
RSMON_WEBAPP_DATA_DIR="$data_dir/webapp" \
|
||||||
|
RSMON_WORKER_ENV_FILE="$env_file" sh -c '
|
||||||
|
set -a
|
||||||
|
. "$RSMON_WORKER_ENV_FILE"
|
||||||
|
set +a
|
||||||
|
unset RSMON_WORKER_ENV_FILE
|
||||||
|
exec "$@"
|
||||||
|
' sh "$@"
|
||||||
|
}
|
||||||
|
|
||||||
|
process_up() {
|
||||||
|
# True when the given pid is a live, non-zombie process. A zombie
|
||||||
|
# still answers kill -0 (its task struct exists until reaped), and in
|
||||||
|
# a container where PID 1 does not reap children a killed worker can
|
||||||
|
# linger as a zombie for a long time, so the /proc state must be
|
||||||
|
# checked explicitly.
|
||||||
|
[ -n "$1" ] || return 1
|
||||||
|
kill -0 "$1" 2>/dev/null || return 1
|
||||||
|
state="$(awk '{print $3}' "/proc/$1/stat" 2>/dev/null || true)"
|
||||||
|
[ "$state" != "Z" ] || return 1
|
||||||
|
}
|
||||||
|
|
||||||
|
process_is_worker() {
|
||||||
|
# True when the given pid's executable is (or was, before an atomic
|
||||||
|
# binary swap) the configured worker binary, so a stale or recycled
|
||||||
|
# pid file can never make the supervisor kill an unrelated process.
|
||||||
|
[ -n "$1" ] || return 1
|
||||||
|
exe="$(readlink "/proc/$1/exe" 2>/dev/null || true)"
|
||||||
|
case "$exe" in
|
||||||
|
"$binary"|"$binary (deleted)") return 0 ;;
|
||||||
|
esac
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
|
||||||
|
stop_process() {
|
||||||
|
if [ -f "$pid_file" ]; then
|
||||||
|
pid="$(cat "$pid_file" 2>/dev/null || true)"
|
||||||
|
if [ -n "$pid" ] && process_is_worker "$pid" && process_up "$pid"; then
|
||||||
|
kill "$pid" 2>/dev/null || true
|
||||||
|
i=0
|
||||||
|
while [ "$i" -lt 10 ]; do
|
||||||
|
if ! process_up "$pid"; then
|
||||||
|
break
|
||||||
|
fi
|
||||||
|
sleep 1
|
||||||
|
i=$((i + 1))
|
||||||
|
done
|
||||||
|
kill -9 "$pid" 2>/dev/null || true
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
rm -f "$pid_file"
|
||||||
|
}
|
||||||
|
|
||||||
|
start_process() {
|
||||||
|
stop_process
|
||||||
|
mkdir -p "$data_dir"
|
||||||
|
run_env sh -c 'nohup "$1" >>"$2" 2>&1 & echo $! > "$3"' sh "$binary" "$log_file" "$pid_file"
|
||||||
|
}
|
||||||
|
|
||||||
|
process_alive() {
|
||||||
|
[ -f "$pid_file" ] || return 1
|
||||||
|
pid="$(cat "$pid_file" 2>/dev/null || true)"
|
||||||
|
process_is_worker "$pid" || return 1
|
||||||
|
process_up "$pid"
|
||||||
|
}
|
||||||
|
|
||||||
|
worker_come_up() {
|
||||||
|
# Wait up to 10s for the freshly-started worker to become a live,
|
||||||
|
# non-zombie process running the configured binary. The recorded pid
|
||||||
|
# starts as the nohup/sh child before it execs the worker, so early
|
||||||
|
# readlink checks can transiently see the interpreter; this retry both
|
||||||
|
# tolerates that exec window and catches genuine immediate deaths.
|
||||||
|
i=0
|
||||||
|
while [ "$i" -lt 10 ]; do
|
||||||
|
if process_alive; then
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
sleep 1
|
||||||
|
i=$((i + 1))
|
||||||
|
done
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
|
||||||
|
health_ok() {
|
||||||
|
run_env "$binary" liveness >/dev/null 2>&1
|
||||||
|
}
|
||||||
|
|
||||||
|
svc_active() {
|
||||||
|
# The health loop verifies the service is active through the
|
||||||
|
# supervisor (never through a pid file for the init-managed paths).
|
||||||
|
case "$supervisor" in
|
||||||
|
systemd) systemctl is-active --quiet "$unit_name" ;;
|
||||||
|
openrc) rc-service "$rc_name" status >/dev/null 2>&1 ;;
|
||||||
|
none) process_alive ;;
|
||||||
|
esac
|
||||||
|
}
|
||||||
|
|
||||||
|
stop_svc() {
|
||||||
|
case "$supervisor" in
|
||||||
|
systemd) systemctl stop "$unit_name" >/dev/null 2>&1 || true ;;
|
||||||
|
openrc) rc-service "$rc_name" stop >/dev/null 2>&1 || true ;;
|
||||||
|
none) stop_process ;;
|
||||||
|
esac
|
||||||
|
}
|
||||||
|
|
||||||
|
restart_svc() {
|
||||||
|
case "$supervisor" in
|
||||||
|
systemd)
|
||||||
|
systemctl daemon-reload >/dev/null 2>&1
|
||||||
|
if ! systemctl restart "$unit_name" >/dev/null 2>&1; then
|
||||||
|
echo "start failure: systemctl restart $unit_name failed" >&2
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
if ! systemctl is-active --quiet "$unit_name"; then
|
||||||
|
echo "start failure: unit $unit_name is not active" >&2
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
;;
|
||||||
|
openrc)
|
||||||
|
if ! rc-service "$rc_name" restart >/dev/null 2>&1; then
|
||||||
|
echo "start failure: rc-service restart $rc_name failed" >&2
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
if ! rc-service "$rc_name" status >/dev/null 2>&1; then
|
||||||
|
echo "start failure: service $rc_name is not running" >&2
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
;;
|
||||||
|
none)
|
||||||
|
start_process
|
||||||
|
if ! worker_come_up; then
|
||||||
|
echo "start failure: worker process did not stay alive" >&2
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
i=0
|
||||||
|
while [ "$i" -lt 30 ]; do
|
||||||
|
if health_ok; then
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
# A worker that stops while the health gate is still probing is a
|
||||||
|
# start failure (it never became a stable service), not merely an
|
||||||
|
# unhealthy-but-running one.
|
||||||
|
if ! svc_active; then
|
||||||
|
echo "start failure: worker service is not active" >&2
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
sleep 1
|
||||||
|
i=$((i + 1))
|
||||||
|
done
|
||||||
|
echo "health failure: worker did not answer /healthz" >&2
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
|
||||||
|
# --- unit enable helpers -------------------------------------------------
|
||||||
|
# unit_was_enabled records whether the unit referenced by the current
|
||||||
|
# params is enabled (best-effort; a missing init leaves it disabled).
|
||||||
|
unit_was_enabled() {
|
||||||
|
case "$supervisor" in
|
||||||
|
systemd) systemctl is-enabled "$unit_name" >/dev/null 2>&1 && unit_enabled=1 ;;
|
||||||
|
openrc) [ -e "/etc/runlevels/default/$rc_name" ] && unit_enabled=1 ;;
|
||||||
|
esac
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
# set_unit_enabled "$1" enables (1) or disables (0) the unit.
|
||||||
|
set_unit_enabled() {
|
||||||
|
case "$supervisor" in
|
||||||
|
systemd)
|
||||||
|
systemctl daemon-reload >/dev/null 2>&1 || true
|
||||||
|
if [ "$1" -eq 1 ]; then
|
||||||
|
systemctl enable "$unit_name" >/dev/null 2>&1 || true
|
||||||
|
else
|
||||||
|
systemctl disable "$unit_name" >/dev/null 2>&1 || true
|
||||||
|
fi
|
||||||
|
;;
|
||||||
|
openrc)
|
||||||
|
if [ "$1" -eq 1 ]; then
|
||||||
|
rc-update add "$rc_name" default >/dev/null 2>&1 || true
|
||||||
|
else
|
||||||
|
rc-update del "$rc_name" default >/dev/null 2>&1 || true
|
||||||
|
fi
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
# --- activation lock ------------------------------------------------------
|
||||||
|
# Prevent concurrent activations on the same host from racing the
|
||||||
|
# deployed-state mutation. A stale lock (left over from a SIGKILL'd run
|
||||||
|
# whose pid is no longer alive) is broken automatically.
|
||||||
|
acquire_lock() {
|
||||||
|
mkdir -p "$data_dir"
|
||||||
|
if [ -d "$lock_dir" ]; then
|
||||||
|
lockpid="$(cat "$lock_dir/pid" 2>/dev/null || true)"
|
||||||
|
if [ -n "$lockpid" ] && ! kill -0 "$lockpid" 2>/dev/null; then
|
||||||
|
rm -rf "$lock_dir" 2>/dev/null || true
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
i=0
|
||||||
|
while [ "$i" -lt 60 ]; do
|
||||||
|
if mkdir "$lock_dir" 2>/dev/null; then
|
||||||
|
chmod 0700 "$lock_dir" 2>/dev/null || true
|
||||||
|
echo $$ > "$lock_dir/pid" 2>/dev/null || true
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
sleep 1
|
||||||
|
i=$((i + 1))
|
||||||
|
done
|
||||||
|
echo "another rsmon-worker activation is in progress ($lock_dir)" >&2
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
release_lock() {
|
||||||
|
rm -rf "$lock_dir" 2>/dev/null || true
|
||||||
|
}
|
||||||
|
|
||||||
|
if ! acquire_lock; then
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
# Any early failure (before the rollback trap is armed) still releases the
|
||||||
|
# lock.
|
||||||
|
trap 'release_lock' EXIT
|
||||||
|
|
||||||
|
# --- interrupted-run recovery --------------------------------------------
|
||||||
|
# A previous activation that was killed mid-flight (SSH drop, SIGKILL)
|
||||||
|
# leaves its backup marker behind. Restore that snapshot before the fresh
|
||||||
|
# activation runs, so the host never stays half-activated and the next run
|
||||||
|
# starts from a consistent prior state.
|
||||||
|
unit_enabled=0
|
||||||
|
recover_backup() {
|
||||||
|
if [ ! -f "$backup_dir/.marker" ]; then
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
echo "recovering interrupted activation from $backup_dir" >&2
|
||||||
|
if [ -f "$backup_dir/binary" ]; then
|
||||||
|
mkdir -p "$(dirname "$binary")" || true
|
||||||
|
cp -p "$backup_dir/binary" "$binary" || true
|
||||||
|
fi
|
||||||
|
if [ -f "$backup_dir/worker.env" ]; then
|
||||||
|
mkdir -p "$config_dir" || true
|
||||||
|
cp -p "$backup_dir/worker.env" "$env_file" || true
|
||||||
|
fi
|
||||||
|
if [ -n "$unit_file" ] && [ -f "$backup_dir/unit" ]; then
|
||||||
|
mkdir -p "$(dirname "$unit_file")" || true
|
||||||
|
cp -p "$backup_dir/unit" "$unit_file" || true
|
||||||
|
chmod "$unit_mode" "$unit_file" || true
|
||||||
|
fi
|
||||||
|
if [ "$(cat "$backup_dir/.marker" 2>/dev/null || true)" = "1" ]; then
|
||||||
|
unit_enabled=1
|
||||||
|
fi
|
||||||
|
set_unit_enabled "$unit_enabled" || true
|
||||||
|
if [ -f "$backup_dir/binary" ] && [ -f "$backup_dir/worker.env" ] && [ "$no_start" -ne 1 ]; then
|
||||||
|
restart_svc || true
|
||||||
|
fi
|
||||||
|
rm -rf "$backup_dir" || true
|
||||||
|
}
|
||||||
|
recover_backup
|
||||||
|
|
||||||
|
# --- staged binary validation (no state touched) -------------------------
|
||||||
|
# A corrupt or missing staging build aborts here and leaves the prior
|
||||||
|
# install completely untouched (the rollback trap is not armed yet).
|
||||||
|
"$stage" --version >/dev/null
|
||||||
|
|
||||||
|
# --- snapshot the prior install ------------------------------------------
|
||||||
|
rm -rf "$backup_dir"
|
||||||
|
mkdir -p "$backup_dir"
|
||||||
|
had_binary=0
|
||||||
|
had_env=0
|
||||||
|
had_unit=0
|
||||||
|
unit_enabled=0
|
||||||
|
if [ -e "$binary" ]; then
|
||||||
|
cp -p "$binary" "$backup_dir/binary"
|
||||||
|
had_binary=1
|
||||||
|
fi
|
||||||
|
if [ -e "$env_file" ]; then
|
||||||
|
cp -p "$env_file" "$backup_dir/worker.env"
|
||||||
|
had_env=1
|
||||||
|
fi
|
||||||
|
if [ -n "$unit_file" ] && [ -e "$unit_file" ]; then
|
||||||
|
had_unit=1
|
||||||
|
cp -p "$unit_file" "$backup_dir/unit"
|
||||||
|
unit_was_enabled
|
||||||
|
fi
|
||||||
|
# The marker is written only after the snapshot copies so recovery never
|
||||||
|
# sees a partial snapshot; it records whether the prior unit was enabled.
|
||||||
|
printf '%s\n' "$unit_enabled" > "$backup_dir/.marker"
|
||||||
|
|
||||||
|
# --- rollback trap --------------------------------------------------------
|
||||||
|
# Any failure from here on restores the snapshot (preserving binary/env/unit
|
||||||
|
# metadata), restores the prior enable state (or disables a freshly-installed
|
||||||
|
# unit on a fresh install), and brings the prior service back up. The trap
|
||||||
|
# also fires on HUP/INT/TERM so an interrupted run rolls back instead of
|
||||||
|
# leaving a half-activated state.
|
||||||
|
rolled_back=0
|
||||||
|
rollback() {
|
||||||
|
[ "$rolled_back" -eq 1 ] && return 0
|
||||||
|
rolled_back=1
|
||||||
|
echo "rsmon-worker activation failed; restoring the prior install" >&2
|
||||||
|
stop_svc || true
|
||||||
|
rm -f "$binary.new" "$env_file.new" "$unit_file.new" "$pid_file" || true
|
||||||
|
if [ "$had_binary" -eq 1 ]; then
|
||||||
|
cp -p "$backup_dir/binary" "$binary" || true
|
||||||
|
elif [ -e "$binary" ]; then
|
||||||
|
rm -f "$binary" || true
|
||||||
|
fi
|
||||||
|
if [ "$had_env" -eq 1 ]; then
|
||||||
|
cp -p "$backup_dir/worker.env" "$env_file" || true
|
||||||
|
chmod 0600 "$env_file" || true
|
||||||
|
elif [ -e "$env_file" ]; then
|
||||||
|
rm -f "$env_file" || true
|
||||||
|
fi
|
||||||
|
if [ -n "$unit_file" ]; then
|
||||||
|
if [ "$had_unit" -eq 1 ]; then
|
||||||
|
cp -p "$backup_dir/unit" "$unit_file" || true
|
||||||
|
chmod "$unit_mode" "$unit_file" || true
|
||||||
|
set_unit_enabled "$unit_enabled" || true
|
||||||
|
else
|
||||||
|
set_unit_enabled 0 || true
|
||||||
|
rm -f "$unit_file" || true
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
rm -f "$env_tmp" "$unit_tmp" || true
|
||||||
|
rm -rf "$backup_dir" || true
|
||||||
|
if [ "$had_binary" -eq 1 ] && [ "$had_env" -eq 1 ] && [ "$no_start" -ne 1 ]; then
|
||||||
|
restart_svc || true
|
||||||
|
fi
|
||||||
|
release_lock
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
trap rollback EXIT HUP INT TERM
|
||||||
|
|
||||||
|
# --- atomic binary install ------------------------------------------------
|
||||||
|
mkdir -p "$(dirname "$binary")"
|
||||||
|
install -m 0755 "$stage" "$binary.new"
|
||||||
|
mv -f "$binary.new" "$binary"
|
||||||
|
|
||||||
|
# --- atomic env install (secrets, mode 0600) ------------------------------
|
||||||
|
mkdir -p "$config_dir"
|
||||||
|
chmod 0750 "$config_dir"
|
||||||
|
install -m 0600 "$env_tmp" "$env_file.new"
|
||||||
|
mv -f "$env_file.new" "$env_file"
|
||||||
|
rm -f "$env_tmp"
|
||||||
|
|
||||||
|
# --- data directory -------------------------------------------------------
|
||||||
|
mkdir -p "$data_dir" "$data_dir/webapp"
|
||||||
|
chmod 0755 "$data_dir" "$data_dir/webapp"
|
||||||
|
|
||||||
|
# --- service definition ---------------------------------------------------
|
||||||
|
if [ -n "$unit_file" ] && [ -f "$unit_tmp" ]; then
|
||||||
|
mkdir -p "$(dirname "$unit_file")"
|
||||||
|
install -m "$unit_mode" "$unit_tmp" "$unit_file.new"
|
||||||
|
mv -f "$unit_file.new" "$unit_file"
|
||||||
|
rm -f "$unit_tmp"
|
||||||
|
set_unit_enabled 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
# --- start and verify -----------------------------------------------------
|
||||||
|
# A failed start or a worker that does not answer /healthz triggers the
|
||||||
|
# rollback trap.
|
||||||
|
if [ "$no_start" -ne 1 ]; then
|
||||||
|
if ! restart_svc; then
|
||||||
|
echo "rsmon-worker activation failed: process or /healthz verification failed" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
else
|
||||||
|
stop_svc || true
|
||||||
|
fi
|
||||||
|
|
||||||
|
# --- success --------------------------------------------------------------
|
||||||
|
rm -rf "$backup_dir"
|
||||||
|
rm -f "$env_tmp" "$unit_tmp"
|
||||||
|
release_lock
|
||||||
|
trap - EXIT HUP INT TERM
|
||||||
|
if [ "$no_start" -eq 1 ]; then
|
||||||
|
echo "rsmon-worker activated: binary=$binary supervisor=$supervisor unit=${unit_file:-none} started=no"
|
||||||
|
else
|
||||||
|
echo "rsmon-worker activated: binary=$binary supervisor=$supervisor unit=${unit_file:-none} started=yes"
|
||||||
|
fi
|
||||||
|
exit 0
|
||||||
|
`
|
||||||
|
|
||||||
|
// activateParams are the resolved, controller-validated values fed into
|
||||||
|
// the remote activation script.
|
||||||
|
type activateParams struct {
|
||||||
|
Stage string
|
||||||
|
Binary string
|
||||||
|
ConfigDir string
|
||||||
|
EnvFile string
|
||||||
|
EnvTmp string
|
||||||
|
DataDir string
|
||||||
|
UnitTmp string
|
||||||
|
UnitFile string
|
||||||
|
UnitMode string
|
||||||
|
UnitName string // systemd unit name (rsmon-worker.service)
|
||||||
|
RCName string // rc-service name (rsmon-worker)
|
||||||
|
Supervisor string // systemd | openrc | none
|
||||||
|
NoStart string // 0 or 1
|
||||||
|
}
|
||||||
|
|
||||||
|
// activateWorker performs the work-package-4 activation for an already
|
||||||
|
// staged source build over the live SSH executor. It uses the env bytes
|
||||||
|
// rendered once during option normalization, uploads the env and service
|
||||||
|
// definition into a server-created 0700 temp dir (no /tmp symlink/TOCTOU
|
||||||
|
// attack surface), runs the atomic activation script, and records where
|
||||||
|
// and how the worker was activated. Every remote temp file it creates is
|
||||||
|
// removed on success and failure.
|
||||||
|
func (e *sourceExecutor) activateWorker(opts SourceInstallOptions, res *SourceInstallResult) error {
|
||||||
|
a := opts.Activation
|
||||||
|
name := strings.TrimSpace(a.Name)
|
||||||
|
p := resolvePaths(name)
|
||||||
|
|
||||||
|
envData := opts.activationEnv
|
||||||
|
if len(envData) == 0 {
|
||||||
|
return errors.New("activation environment was not rendered (internal error)")
|
||||||
|
}
|
||||||
|
|
||||||
|
unitContent, unitFile, unitMode := unitContent(res.Detection.InitSystem, p)
|
||||||
|
svcName := strings.TrimSuffix(p.unitName, ".service")
|
||||||
|
|
||||||
|
supOut, err := e.runPlain("sh -c " + shellQuote(runningInitScript))
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("detect running init system: %w", err)
|
||||||
|
}
|
||||||
|
supervisor := normalizeSupervisor(string(supOut))
|
||||||
|
|
||||||
|
// Create a server-side 0700 temp dir owned by the SSH user, then
|
||||||
|
// upload the env/unit into it. mktemp -d produces an unpredictable,
|
||||||
|
// private path, so a hostile local user cannot pre-create a symlink at
|
||||||
|
// a predictable /tmp name (the classic upload TOCTOU). The token
|
||||||
|
// never appears in argv or logs: it travels only as base64 over the
|
||||||
|
// session stdin and later only inside the mode-0600 env file.
|
||||||
|
out, err := e.runPlain("d=$(mktemp -d /tmp/rsmon-worker-act.XXXXXX) && chmod 0700 \"$d\" && echo \"$d\"")
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("create secure upload directory: %w", err)
|
||||||
|
}
|
||||||
|
secureDir := strings.TrimSpace(string(out))
|
||||||
|
if secureDir == "" || !strings.HasPrefix(secureDir, "/tmp/") {
|
||||||
|
return errors.New("remote returned an invalid secure upload directory")
|
||||||
|
}
|
||||||
|
envTmp := secureDir + "/worker.env"
|
||||||
|
unitTmp := secureDir + "/unit"
|
||||||
|
if err := uploadBytes(e.client, envData, envTmp, 0o600); err != nil {
|
||||||
|
return fmt.Errorf("upload worker environment: %w", err)
|
||||||
|
}
|
||||||
|
if unitContent != "" {
|
||||||
|
if err := uploadBytes(e.client, []byte(unitContent), unitTmp, 0o644); err != nil {
|
||||||
|
return fmt.Errorf("upload service definition: %w", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
defer func() {
|
||||||
|
_ = runRemote(e.client, "rm -rf -- "+shellQuote(secureDir), nil)
|
||||||
|
}()
|
||||||
|
|
||||||
|
noStart := "0"
|
||||||
|
if a.NoStart {
|
||||||
|
noStart = "1"
|
||||||
|
}
|
||||||
|
script := activateScript(activateParams{
|
||||||
|
Stage: res.StageBinary,
|
||||||
|
Binary: p.binary,
|
||||||
|
ConfigDir: p.configDir,
|
||||||
|
EnvFile: p.envFile,
|
||||||
|
EnvTmp: envTmp,
|
||||||
|
DataDir: p.dataDir,
|
||||||
|
UnitTmp: unitTmp,
|
||||||
|
UnitFile: unitFile,
|
||||||
|
UnitMode: unitMode,
|
||||||
|
UnitName: p.unitName,
|
||||||
|
RCName: svcName,
|
||||||
|
Supervisor: supervisor,
|
||||||
|
NoStart: noStart,
|
||||||
|
})
|
||||||
|
if _, err := e.runPrivileged("sh -c " + shellQuote(script)); err != nil {
|
||||||
|
return fmt.Errorf("activate worker service: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
res.Activation = &ActivationResult{
|
||||||
|
Binary: p.binary,
|
||||||
|
ConfigDir: p.configDir,
|
||||||
|
EnvFile: p.envFile,
|
||||||
|
DataDir: p.dataDir,
|
||||||
|
UnitFile: unitFile,
|
||||||
|
UnitName: p.unitName,
|
||||||
|
Supervisor: supervisor,
|
||||||
|
Started: !a.NoStart,
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// validateRenderedEnv runs the strict environment-file parser over the
|
||||||
|
// rendered env bytes in memory so the exact bytes written remotely pass
|
||||||
|
// the same gate as a user-supplied file, without re-reading a file.
|
||||||
|
func validateRenderedEnv(data []byte) error {
|
||||||
|
_, err := parseEnvironmentContent(data)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// openrcInitFor renders the native OpenRC init script for an instance. It
|
||||||
|
// mirrors the hardening of the systemd unit (data dir, env file, webapp
|
||||||
|
// data dir) using OpenRC conventions; it is written for real hosts where
|
||||||
|
// openrc is the running init, while containers that cannot run openrc
|
||||||
|
// fall back to the embedded supervisor.
|
||||||
|
func openrcInitFor(p paths) string {
|
||||||
|
description := "RSMon distributed monitoring worker"
|
||||||
|
svcName := strings.TrimSuffix(p.unitName, ".service")
|
||||||
|
if p.name != "" {
|
||||||
|
description += " (" + p.name + ")"
|
||||||
|
}
|
||||||
|
return fmt.Sprintf(`#!/sbin/openrc-run
|
||||||
|
# Managed by the rsmon-worker source installer; do not edit by hand.
|
||||||
|
|
||||||
|
name=%s
|
||||||
|
description=%s
|
||||||
|
command=%s
|
||||||
|
command_background=true
|
||||||
|
pidfile=%s/worker.pid
|
||||||
|
output_log=%s/worker.log
|
||||||
|
error_log=%s/worker.log
|
||||||
|
|
||||||
|
depend() {
|
||||||
|
need net
|
||||||
|
}
|
||||||
|
|
||||||
|
start_pre() {
|
||||||
|
checkpath --directory --mode 0755 --owner root:root %s %s/webapp
|
||||||
|
if [ -f %s ]; then
|
||||||
|
# OpenRC runs the command with the init script's environment, so
|
||||||
|
# the env-file variables must be exported (set -a) or the worker
|
||||||
|
# would start without them.
|
||||||
|
set -a
|
||||||
|
. %s
|
||||||
|
set +a
|
||||||
|
fi
|
||||||
|
export HOME=%s
|
||||||
|
export RSMON_WEBAPP_DATA_DIR=%s/webapp
|
||||||
|
}
|
||||||
|
`, svcName, description, p.binary, p.dataDir, p.dataDir, p.dataDir,
|
||||||
|
p.dataDir, p.dataDir, p.envFile, p.envFile, p.dataDir, p.dataDir)
|
||||||
|
}
|
||||||
438
internal/installer/sourceactivate_shell_test.go
Обычный файл
438
internal/installer/sourceactivate_shell_test.go
Обычный файл
@@ -0,0 +1,438 @@
|
|||||||
|
package installer
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os"
|
||||||
|
"os/exec"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
// This file exercises the rendered activation script against stub
|
||||||
|
// systemd/OpenRC implementations ("targeted shell fixture tests"). Real
|
||||||
|
// systemd and OpenRC cannot run inside the Docker/OpenSSH harness (the
|
||||||
|
// container PID 1 is sshd), so these tests execute the actual remote
|
||||||
|
// script on the host with fake `systemctl`, `rc-service`, and `rc-update`
|
||||||
|
// binaries that simulate an init-managed host. This covers the unit
|
||||||
|
// install/enable, the supervisor-driven restart, the `is-active`-based
|
||||||
|
// health loop, the enable-state rollback, the activation lock, and the
|
||||||
|
// interrupted-run recovery paths that the E2E fixtures cannot reach.
|
||||||
|
//
|
||||||
|
// Residual limitation: the stub init tools verify command flow and state
|
||||||
|
// transitions, not the real systemd/OpenRC unit semantics; a real
|
||||||
|
// init-system smoke test remains out of scope for the container harness.
|
||||||
|
|
||||||
|
// systemctlStub records every invocation and simulates unit state in a
|
||||||
|
// fake state directory: `is-active`/`is-enabled` consult marker files,
|
||||||
|
// `restart` marks the unit active (or fails when the fail-restart marker
|
||||||
|
// is present), and `enable`/`disable` toggle the enabled marker.
|
||||||
|
const systemctlStub = `#!/bin/sh
|
||||||
|
echo "$*" >> "$SYSCTL_LOG"
|
||||||
|
action="$1"
|
||||||
|
shift
|
||||||
|
unit=""
|
||||||
|
for a in "$@"; do
|
||||||
|
case "$a" in --*) continue ;; esac
|
||||||
|
unit="$a"
|
||||||
|
break
|
||||||
|
done
|
||||||
|
case "$action" in
|
||||||
|
is-active) [ -f "$FAKE_SYSTEMD_DIR/$unit.active" ] || exit 3 ;;
|
||||||
|
is-enabled) [ -f "$FAKE_SYSTEMD_DIR/$unit.enabled" ] || exit 1 ;;
|
||||||
|
enable) touch "$FAKE_SYSTEMD_DIR/$unit.enabled"; exit 0 ;;
|
||||||
|
disable) rm -f "$FAKE_SYSTEMD_DIR/$unit.enabled"; exit 0 ;;
|
||||||
|
daemon-reload) exit 0 ;;
|
||||||
|
restart) [ -f "$FAKE_SYSTEMD_DIR/fail-restart" ] && exit 1
|
||||||
|
touch "$FAKE_SYSTEMD_DIR/$unit.active"; exit 0 ;;
|
||||||
|
stop) rm -f "$FAKE_SYSTEMD_DIR/$unit.active"; exit 0 ;;
|
||||||
|
*) exit 0 ;;
|
||||||
|
esac
|
||||||
|
`
|
||||||
|
|
||||||
|
// rcServiceStub simulates OpenRC service state; rcUpdateStub simulates
|
||||||
|
// runlevel enablement.
|
||||||
|
const rcServiceStub = `#!/bin/sh
|
||||||
|
echo "$*" >> "$RC_LOG"
|
||||||
|
svc="$1"
|
||||||
|
action="$2"
|
||||||
|
case "$action" in
|
||||||
|
restart) [ -f "$FAKE_RC_DIR/fail-restart" ] && exit 1
|
||||||
|
touch "$FAKE_RC_DIR/$svc.active"; exit 0 ;;
|
||||||
|
status) [ -f "$FAKE_RC_DIR/$svc.active" ] || exit 1 ;;
|
||||||
|
stop) rm -f "$FAKE_RC_DIR/$svc.active"; exit 0 ;;
|
||||||
|
*) exit 0 ;;
|
||||||
|
esac
|
||||||
|
`
|
||||||
|
|
||||||
|
const rcUpdateStub = `#!/bin/sh
|
||||||
|
echo "$*" >> "$RCU_LOG"
|
||||||
|
case "$1" in
|
||||||
|
add) touch "$FAKE_RC_DIR/$2.enabled"; exit 0 ;;
|
||||||
|
del) rm -f "$FAKE_RC_DIR/$2.enabled"; exit 0 ;;
|
||||||
|
*) exit 0 ;;
|
||||||
|
esac
|
||||||
|
`
|
||||||
|
|
||||||
|
// workerStub stands in for the built worker binary: it satisfies the
|
||||||
|
// `--version` validation and the `/healthz` liveness probe.
|
||||||
|
const workerStub = `#!/bin/sh
|
||||||
|
if [ "$1" = "--version" ]; then
|
||||||
|
echo "rsmon-worker version=dev commit=deadbeefdead buildDate=2026-01-01T00:00:00Z"
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
if [ "$1" = "liveness" ]; then
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
exit 1
|
||||||
|
`
|
||||||
|
|
||||||
|
// initShellFixture builds a temp "host" with stub init tools and a stub
|
||||||
|
// worker, plus the uploaded env/unit the activation script consumes.
|
||||||
|
type initShellFixture struct {
|
||||||
|
root string
|
||||||
|
binDir string
|
||||||
|
sysdDir string
|
||||||
|
rcDir string
|
||||||
|
|
||||||
|
sysctlLog string
|
||||||
|
rcLog string
|
||||||
|
rcuLog string
|
||||||
|
|
||||||
|
params activateParams
|
||||||
|
}
|
||||||
|
|
||||||
|
func writeExec(t *testing.T, path, content string) {
|
||||||
|
t.Helper()
|
||||||
|
if err := os.WriteFile(path, []byte(content), 0o700); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func newInitShellFixture(t *testing.T, supervisor string) *initShellFixture {
|
||||||
|
t.Helper()
|
||||||
|
root := t.TempDir()
|
||||||
|
fx := &initShellFixture{
|
||||||
|
root: root,
|
||||||
|
binDir: filepath.Join(root, "bin"),
|
||||||
|
sysdDir: filepath.Join(root, "sysd"),
|
||||||
|
rcDir: filepath.Join(root, "rc"),
|
||||||
|
sysctlLog: filepath.Join(root, "sysctl.log"),
|
||||||
|
rcLog: filepath.Join(root, "rc.log"),
|
||||||
|
rcuLog: filepath.Join(root, "rcu.log"),
|
||||||
|
}
|
||||||
|
for _, d := range []string{
|
||||||
|
fx.binDir, fx.sysdDir, fx.rcDir, filepath.Join(root, "upload"),
|
||||||
|
filepath.Join(root, "usr/local/bin"), filepath.Join(root, "etc/rsmon-worker"),
|
||||||
|
} {
|
||||||
|
if err := os.MkdirAll(d, 0o755); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
writeExec(t, filepath.Join(fx.binDir, "systemctl"), systemctlStub)
|
||||||
|
writeExec(t, filepath.Join(fx.binDir, "rc-service"), rcServiceStub)
|
||||||
|
writeExec(t, filepath.Join(fx.binDir, "rc-update"), rcUpdateStub)
|
||||||
|
|
||||||
|
stage := filepath.Join(root, "usr/local/bin/rsmon-worker.stage")
|
||||||
|
writeExec(t, stage, workerStub)
|
||||||
|
|
||||||
|
envTmp := filepath.Join(root, "upload", "worker.env")
|
||||||
|
unitTmp := filepath.Join(root, "upload", "unit")
|
||||||
|
if err := os.WriteFile(envTmp, []byte(
|
||||||
|
"RSMON_URL=https://rsmon.ru\nRSMON_TOKEN=shell-test-token\nWORKER_HOST=127.0.0.1\nWORKER_PORT=27401\n",
|
||||||
|
), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
unitFile := filepath.Join(root, "etc/systemd/system/rsmon-worker.service")
|
||||||
|
unitMode := "0644"
|
||||||
|
if supervisor == "openrc" {
|
||||||
|
unitFile = filepath.Join(root, "etc/init.d/rsmon-worker")
|
||||||
|
unitMode = "0755"
|
||||||
|
if err := os.MkdirAll(filepath.Dir(unitFile), 0o755); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(unitTmp, []byte("# fake openrc unit\n"), 0o755); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
if err := os.MkdirAll(filepath.Dir(unitFile), 0o755); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(unitTmp, []byte("[Unit]\n# fake systemd unit\n"), 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
dataDir := filepath.Join(root, "var/lib/rsmon-worker")
|
||||||
|
fx.params = activateParams{
|
||||||
|
Stage: stage,
|
||||||
|
Binary: filepath.Join(root, "usr/local/bin/rsmon-worker"),
|
||||||
|
ConfigDir: filepath.Join(root, "etc/rsmon-worker"),
|
||||||
|
EnvFile: filepath.Join(root, "etc/rsmon-worker/worker.env"),
|
||||||
|
EnvTmp: envTmp,
|
||||||
|
DataDir: dataDir,
|
||||||
|
UnitTmp: unitTmp,
|
||||||
|
UnitFile: unitFile,
|
||||||
|
UnitMode: unitMode,
|
||||||
|
UnitName: "rsmon-worker.service",
|
||||||
|
RCName: "rsmon-worker",
|
||||||
|
Supervisor: supervisor,
|
||||||
|
NoStart: "0",
|
||||||
|
}
|
||||||
|
return fx
|
||||||
|
}
|
||||||
|
|
||||||
|
// run executes the rendered activation script with the stub init tools
|
||||||
|
// first in PATH.
|
||||||
|
func (fx *initShellFixture) run(t *testing.T) (string, error) {
|
||||||
|
t.Helper()
|
||||||
|
script := activateScript(fx.params)
|
||||||
|
cmd := exec.Command("sh", "-c", script)
|
||||||
|
cmd.Env = append(
|
||||||
|
os.Environ(),
|
||||||
|
"PATH="+fx.binDir+":"+os.Getenv("PATH"),
|
||||||
|
"FAKE_SYSTEMD_DIR="+fx.sysdDir,
|
||||||
|
"FAKE_RC_DIR="+fx.rcDir,
|
||||||
|
"SYSCTL_LOG="+fx.sysctlLog,
|
||||||
|
"RC_LOG="+fx.rcLog,
|
||||||
|
"RCU_LOG="+fx.rcuLog,
|
||||||
|
)
|
||||||
|
out, err := cmd.CombinedOutput()
|
||||||
|
return string(out), err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (fx *initShellFixture) log(t *testing.T, path string) string {
|
||||||
|
t.Helper()
|
||||||
|
b, err := os.ReadFile(path)
|
||||||
|
if err != nil {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return string(b)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (fx *initShellFixture) assertNoLeftovers(t *testing.T) {
|
||||||
|
t.Helper()
|
||||||
|
for _, name := range []string{".rsmon-backup", ".rsmon-activate.lock"} {
|
||||||
|
if _, err := os.Stat(filepath.Join(fx.params.DataDir, name)); !os.IsNotExist(err) {
|
||||||
|
t.Fatalf("%s left behind after activation", name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestActivateShellSystemd runs the real activation script against a stub
|
||||||
|
// systemd host and verifies the init-managed flow: the unit is installed
|
||||||
|
// and enabled, restart goes through systemctl, and the health loop uses
|
||||||
|
// `systemctl is-active` (no pid file is ever created).
|
||||||
|
func TestActivateShellSystemd(t *testing.T) {
|
||||||
|
fx := newInitShellFixture(t, "systemd")
|
||||||
|
out, err := fx.run(t)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("systemd activation failed: %v\n%s", err, out)
|
||||||
|
}
|
||||||
|
if !strings.Contains(out, "rsmon-worker activated") {
|
||||||
|
t.Fatalf("no activation success line:\n%s", out)
|
||||||
|
}
|
||||||
|
unit, err := os.ReadFile(fx.params.UnitFile)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("unit not installed: %v", err)
|
||||||
|
}
|
||||||
|
if !strings.Contains(string(unit), "[Unit]") {
|
||||||
|
t.Fatalf("installed unit has wrong content: %q", unit)
|
||||||
|
}
|
||||||
|
st, err := os.Stat(fx.params.EnvFile)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("env not installed: %v", err)
|
||||||
|
}
|
||||||
|
if st.Mode().Perm() != 0o600 {
|
||||||
|
t.Fatalf("env mode = %v, want 0600", st.Mode().Perm())
|
||||||
|
}
|
||||||
|
log := fx.log(t, fx.sysctlLog)
|
||||||
|
for _, want := range []string{
|
||||||
|
"enable rsmon-worker.service",
|
||||||
|
"restart rsmon-worker.service",
|
||||||
|
"is-active --quiet rsmon-worker.service",
|
||||||
|
} {
|
||||||
|
if !strings.Contains(log, want) {
|
||||||
|
t.Fatalf("systemctl log missing %q:\n%s", want, log)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// The health loop used systemctl is-active, never the embedded
|
||||||
|
// supervisor's pid file.
|
||||||
|
if _, err := os.Stat(filepath.Join(fx.params.DataDir, "worker.pid")); !os.IsNotExist(err) {
|
||||||
|
t.Fatalf("systemd activation created a pid file; the health loop must use systemctl is-active")
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(filepath.Join(fx.sysdDir, "rsmon-worker.service.enabled")); err != nil {
|
||||||
|
t.Fatalf("unit not enabled: %v", err)
|
||||||
|
}
|
||||||
|
fx.assertNoLeftovers(t)
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestActivateShellOpenRC runs the real activation script against a stub
|
||||||
|
// OpenRC host: the init script is installed executable, the runlevel
|
||||||
|
// enablement and rc-service restart/status are used, and the health loop
|
||||||
|
// goes through rc-service (no pid file).
|
||||||
|
func TestActivateShellOpenRC(t *testing.T) {
|
||||||
|
fx := newInitShellFixture(t, "openrc")
|
||||||
|
out, err := fx.run(t)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("openrc activation failed: %v\n%s", err, out)
|
||||||
|
}
|
||||||
|
if !strings.Contains(out, "rsmon-worker activated") {
|
||||||
|
t.Fatalf("no activation success line:\n%s", out)
|
||||||
|
}
|
||||||
|
st, err := os.Stat(fx.params.UnitFile)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("openrc init not installed: %v", err)
|
||||||
|
}
|
||||||
|
if st.Mode().Perm() != 0o755 {
|
||||||
|
t.Fatalf("openrc init mode = %v, want 0755", st.Mode().Perm())
|
||||||
|
}
|
||||||
|
rcLog := fx.log(t, fx.rcLog)
|
||||||
|
for _, want := range []string{"rsmon-worker restart", "rsmon-worker status"} {
|
||||||
|
if !strings.Contains(rcLog, want) {
|
||||||
|
t.Fatalf("rc-service log missing %q:\n%s", want, rcLog)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
rcuLog := fx.log(t, fx.rcuLog)
|
||||||
|
if !strings.Contains(rcuLog, "add rsmon-worker default") {
|
||||||
|
t.Fatalf("rc-update add not issued:\n%s", rcuLog)
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(filepath.Join(fx.rcDir, "rsmon-worker.enabled")); err != nil {
|
||||||
|
t.Fatalf("service not enabled in the runlevel: %v", err)
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(filepath.Join(fx.params.DataDir, "worker.pid")); !os.IsNotExist(err) {
|
||||||
|
t.Fatalf("openrc activation created a pid file; the health loop must use rc-service status")
|
||||||
|
}
|
||||||
|
fx.assertNoLeftovers(t)
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestActivateShellSystemdRollbackFreshDisablesUnit proves a failed
|
||||||
|
// activation on a fresh host disables the newly-enabled unit and removes
|
||||||
|
// the unit file (no dangling enablement).
|
||||||
|
func TestActivateShellSystemdRollbackFreshDisablesUnit(t *testing.T) {
|
||||||
|
fx := newInitShellFixture(t, "systemd")
|
||||||
|
if err := os.WriteFile(filepath.Join(fx.sysdDir, "fail-restart"), []byte(""), 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
out, err := fx.run(t)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("activation succeeded despite restart failure")
|
||||||
|
}
|
||||||
|
if !strings.Contains(out, "start failure") {
|
||||||
|
t.Fatalf("no start-failure classification:\n%s", out)
|
||||||
|
}
|
||||||
|
log := fx.log(t, fx.sysctlLog)
|
||||||
|
if !strings.Contains(log, "disable rsmon-worker.service") {
|
||||||
|
t.Fatalf("fresh rollback did not disable the unit:\n%s", log)
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(fx.params.UnitFile); !os.IsNotExist(err) {
|
||||||
|
t.Fatal("unit file left behind after fresh rollback")
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(filepath.Join(fx.sysdDir, "rsmon-worker.service.enabled")); !os.IsNotExist(err) {
|
||||||
|
t.Fatal("unit still enabled after fresh rollback")
|
||||||
|
}
|
||||||
|
fx.assertNoLeftovers(t)
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestActivateShellSystemdRollbackRestoresEnableState proves a failed
|
||||||
|
// activation on a host with a prior enabled unit restores the unit
|
||||||
|
// byte-for-byte and re-applies the prior enabled state.
|
||||||
|
func TestActivateShellSystemdRollbackRestoresEnableState(t *testing.T) {
|
||||||
|
fx := newInitShellFixture(t, "systemd")
|
||||||
|
if err := os.WriteFile(fx.params.UnitFile, []byte("PRIOR-UNIT-CONTENT"), 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(filepath.Join(fx.sysdDir, "rsmon-worker.service.enabled"), []byte(""), 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(filepath.Join(fx.sysdDir, "fail-restart"), []byte(""), 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
out, err := fx.run(t)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("activation succeeded despite restart failure")
|
||||||
|
}
|
||||||
|
if !strings.Contains(out, "start failure") {
|
||||||
|
t.Fatalf("no start-failure classification:\n%s", out)
|
||||||
|
}
|
||||||
|
unit, err := os.ReadFile(fx.params.UnitFile)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if string(unit) != "PRIOR-UNIT-CONTENT" {
|
||||||
|
t.Fatalf("prior unit not restored: %q", unit)
|
||||||
|
}
|
||||||
|
log := fx.log(t, fx.sysctlLog)
|
||||||
|
if !strings.Contains(log, "enable rsmon-worker.service") {
|
||||||
|
t.Fatalf("prior enabled state not re-applied:\n%s", log)
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(filepath.Join(fx.sysdDir, "rsmon-worker.service.enabled")); err != nil {
|
||||||
|
t.Fatalf("unit not re-enabled after rollback: %v", err)
|
||||||
|
}
|
||||||
|
fx.assertNoLeftovers(t)
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestActivateShellStaleLockBroken proves a leftover lock from a killed
|
||||||
|
// run (dead pid) is broken automatically and the activation proceeds.
|
||||||
|
func TestActivateShellStaleLockBroken(t *testing.T) {
|
||||||
|
fx := newInitShellFixture(t, "systemd")
|
||||||
|
lockDir := filepath.Join(fx.params.DataDir, ".rsmon-activate.lock")
|
||||||
|
if err := os.MkdirAll(lockDir, 0o700); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(filepath.Join(lockDir, "pid"), []byte("999999\n"), 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
out, err := fx.run(t)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("activation failed with a stale lock: %v\n%s", err, out)
|
||||||
|
}
|
||||||
|
if !strings.Contains(out, "rsmon-worker activated") {
|
||||||
|
t.Fatalf("no activation success line:\n%s", out)
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(lockDir); !os.IsNotExist(err) {
|
||||||
|
t.Fatal("lock not released after activation")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestActivateShellRecoveryFromInterruptedBackup proves the interrupted-run
|
||||||
|
// recovery: a leftover backup marker from a killed activation is restored
|
||||||
|
// before the fresh run, so a later validation failure still leaves the
|
||||||
|
// recovered prior install in place.
|
||||||
|
func TestActivateShellRecoveryFromInterruptedBackup(t *testing.T) {
|
||||||
|
fx := newInitShellFixture(t, "none")
|
||||||
|
backupDir := filepath.Join(fx.params.DataDir, ".rsmon-backup")
|
||||||
|
if err := os.MkdirAll(backupDir, 0o700); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(filepath.Join(backupDir, "binary"), []byte("RECOVERED-BINARY"), 0o755); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(filepath.Join(backupDir, "worker.env"), []byte("RSMON_URL=https://rsmon.ru\nRSMON_TOKEN=test\n"), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(filepath.Join(backupDir, ".marker"), []byte("0"), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
// Corrupt the stage so the fresh activation fails at validation (before
|
||||||
|
// the fresh snapshot), proving the recovered files were already in place.
|
||||||
|
if err := os.WriteFile(fx.params.Stage, []byte("not a script"), 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
out, err := fx.run(t)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("activation succeeded with a corrupt stage")
|
||||||
|
}
|
||||||
|
if !strings.Contains(out, "recovering interrupted activation") {
|
||||||
|
t.Fatalf("recovery did not run:\n%s", out)
|
||||||
|
}
|
||||||
|
b, err := os.ReadFile(fx.params.Binary)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("recovered binary not restored: %v", err)
|
||||||
|
}
|
||||||
|
if string(b) != "RECOVERED-BINARY" {
|
||||||
|
t.Fatalf("recovered binary content = %q", b)
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(backupDir); !os.IsNotExist(err) {
|
||||||
|
t.Fatal("backup not consumed by recovery")
|
||||||
|
}
|
||||||
|
}
|
||||||
409
internal/installer/sourceactivate_test.go
Обычный файл
409
internal/installer/sourceactivate_test.go
Обычный файл
@@ -0,0 +1,409 @@
|
|||||||
|
package installer
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"rocketgit.ru/rsmon/worker/internal/sshinstall"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestActivateScriptMarkers(t *testing.T) {
|
||||||
|
script := activateScript(activateParams{
|
||||||
|
Stage: "/opt/rsmon-worker-src/rsmon-worker",
|
||||||
|
Binary: "/usr/local/bin/rsmon-worker",
|
||||||
|
ConfigDir: "/etc/rsmon-worker",
|
||||||
|
EnvFile: "/etc/rsmon-worker/worker.env",
|
||||||
|
EnvTmp: "/tmp/rsmon-worker-abc.env",
|
||||||
|
DataDir: "/var/lib/rsmon-worker",
|
||||||
|
UnitTmp: "/tmp/rsmon-worker-abc.service",
|
||||||
|
UnitFile: "/etc/systemd/system/rsmon-worker.service",
|
||||||
|
UnitMode: "0644",
|
||||||
|
UnitName: "rsmon-worker.service",
|
||||||
|
RCName: "rsmon-worker",
|
||||||
|
Supervisor: "none",
|
||||||
|
NoStart: "0",
|
||||||
|
})
|
||||||
|
for _, want := range []string{
|
||||||
|
"set -eu",
|
||||||
|
// staged binary validated before any state is touched
|
||||||
|
"\"$stage\" --version >/dev/null",
|
||||||
|
// snapshot + rollback
|
||||||
|
"backup_dir=\"$data_dir/.rsmon-backup\"",
|
||||||
|
"rm -rf \"$backup_dir\"",
|
||||||
|
"cp -p \"$binary\" \"$backup_dir/binary\"",
|
||||||
|
"cp -p \"$env_file\" \"$backup_dir/worker.env\"",
|
||||||
|
"printf '%s\\n' \"$unit_enabled\" > \"$backup_dir/.marker\"",
|
||||||
|
"trap rollback EXIT HUP INT TERM",
|
||||||
|
"restoring the prior install",
|
||||||
|
// atomic installs with the right perms
|
||||||
|
"install -m 0755 \"$stage\" \"$binary.new\"",
|
||||||
|
"mv -f \"$binary.new\" \"$binary\"",
|
||||||
|
"install -m 0600 \"$env_tmp\" \"$env_file.new\"",
|
||||||
|
"mv -f \"$env_file.new\" \"$env_file\"",
|
||||||
|
"chmod 0750 \"$config_dir\"",
|
||||||
|
"mkdir -p \"$data_dir\" \"$data_dir/webapp\"",
|
||||||
|
"install -m \"$unit_mode\" \"$unit_tmp\" \"$unit_file.new\"",
|
||||||
|
"mv -f \"$unit_file.new\" \"$unit_file\"",
|
||||||
|
// supervisor + pid + health verification
|
||||||
|
"pid_file=\"$data_dir/worker.pid\"",
|
||||||
|
"env -i PATH=\"/usr/bin:/bin:/sbin:/usr/sbin\"",
|
||||||
|
"RSMON_WORKER_ENV_FILE=\"$env_file\"",
|
||||||
|
". \"$RSMON_WORKER_ENV_FILE\"",
|
||||||
|
"exec \"$@\"",
|
||||||
|
"nohup \"$1\" >>\"$2\" 2>&1 & echo $! > \"$3\"",
|
||||||
|
"run_env \"$binary\" liveness >/dev/null 2>&1",
|
||||||
|
"while [ \"$i\" -lt 30 ]; do",
|
||||||
|
"rsmon-worker activated:",
|
||||||
|
// activation lock + interrupted-run recovery
|
||||||
|
"lock_dir=\"$data_dir/.rsmon-activate.lock\"",
|
||||||
|
"acquire_lock",
|
||||||
|
"another rsmon-worker activation is in progress",
|
||||||
|
"recovering interrupted activation from $backup_dir",
|
||||||
|
".marker",
|
||||||
|
// pid belongs to the expected binary before kill
|
||||||
|
"readlink \"/proc/$1/exe\"",
|
||||||
|
"\"$binary (deleted)\"",
|
||||||
|
"process_is_worker \"$pid\"",
|
||||||
|
// rollback preserves metadata and enable state
|
||||||
|
"cp -p \"$backup_dir/binary\" \"$binary\"",
|
||||||
|
"cp -p \"$backup_dir/worker.env\" \"$env_file\"",
|
||||||
|
"set_unit_enabled \"$unit_enabled\"",
|
||||||
|
"systemctl is-enabled",
|
||||||
|
// temp/backup cleanup on success
|
||||||
|
"rm -rf \"$backup_dir\"",
|
||||||
|
"release_lock",
|
||||||
|
"trap - EXIT HUP INT TERM",
|
||||||
|
} {
|
||||||
|
if !strings.Contains(script, want) {
|
||||||
|
t.Fatalf("activate script missing %q:\n%s", want, script)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// The rollback must be armed only after the snapshot so a corrupt
|
||||||
|
// staging binary (validated first) never triggers a destructive
|
||||||
|
// rollback of the prior install.
|
||||||
|
if !strings.Contains(script, "\"$stage\" --version") || !strings.Contains(script, "trap rollback EXIT HUP INT TERM") {
|
||||||
|
t.Fatalf("activate script must validate the staged binary before arming rollback:\n%s", script)
|
||||||
|
}
|
||||||
|
stageCheck := strings.Index(script, "\"$stage\" --version")
|
||||||
|
trapIdx := strings.Index(script, "trap rollback EXIT HUP INT TERM")
|
||||||
|
if stageCheck < 0 || trapIdx < stageCheck {
|
||||||
|
t.Fatalf("staged-binary validation must precede the rollback trap:\n%s", script)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestActivateScriptSystemdSupervisor(t *testing.T) {
|
||||||
|
script := activateScript(activateParams{
|
||||||
|
Stage: "/opt/rsmon-worker-src/rsmon-worker",
|
||||||
|
Binary: "/usr/local/bin/rsmon-worker",
|
||||||
|
ConfigDir: "/etc/rsmon-worker",
|
||||||
|
EnvFile: "/etc/rsmon-worker/worker.env",
|
||||||
|
EnvTmp: "/tmp/e.env",
|
||||||
|
DataDir: "/var/lib/rsmon-worker",
|
||||||
|
UnitTmp: "/tmp/u.service",
|
||||||
|
UnitFile: "/etc/systemd/system/rsmon-worker.service",
|
||||||
|
UnitMode: "0644",
|
||||||
|
UnitName: "rsmon-worker.service",
|
||||||
|
RCName: "rsmon-worker",
|
||||||
|
Supervisor: "systemd",
|
||||||
|
NoStart: "0",
|
||||||
|
})
|
||||||
|
for _, want := range []string{
|
||||||
|
"supervisor=systemd",
|
||||||
|
"systemctl daemon-reload",
|
||||||
|
"systemctl restart \"$unit_name\"",
|
||||||
|
"systemctl is-active --quiet \"$unit_name\"",
|
||||||
|
"systemctl enable \"$unit_name\"",
|
||||||
|
"svc_active()",
|
||||||
|
"start failure: systemctl restart $unit_name failed",
|
||||||
|
"start failure: worker service is not active",
|
||||||
|
} {
|
||||||
|
if !strings.Contains(script, want) {
|
||||||
|
t.Fatalf("systemd supervisor missing %q:\n%s", want, script)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// The systemd health loop must use `systemctl is-active`, never the
|
||||||
|
// embedded supervisor's pid file.
|
||||||
|
if !strings.Contains(script, "systemd) systemctl is-active --quiet \"$unit_name\"") {
|
||||||
|
t.Fatalf("systemd health loop must use systemctl is-active:\n%s", script)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestActivateScriptOpenRCSupervisor(t *testing.T) {
|
||||||
|
script := activateScript(activateParams{
|
||||||
|
Stage: "/opt/rsmon-worker-src/rsmon-worker",
|
||||||
|
Binary: "/usr/local/bin/rsmon-worker",
|
||||||
|
ConfigDir: "/etc/rsmon-worker",
|
||||||
|
EnvFile: "/etc/rsmon-worker/worker.env",
|
||||||
|
EnvTmp: "/tmp/e.env",
|
||||||
|
DataDir: "/var/lib/rsmon-worker",
|
||||||
|
UnitTmp: "/tmp/u",
|
||||||
|
UnitFile: "/etc/init.d/rsmon-worker",
|
||||||
|
UnitMode: "0755",
|
||||||
|
UnitName: "rsmon-worker.service",
|
||||||
|
RCName: "rsmon-worker",
|
||||||
|
Supervisor: "openrc",
|
||||||
|
NoStart: "0",
|
||||||
|
})
|
||||||
|
for _, want := range []string{
|
||||||
|
"rc-service \"$rc_name\" restart",
|
||||||
|
"rc-service \"$rc_name\" status",
|
||||||
|
"rc-update add \"$rc_name\" default",
|
||||||
|
"start failure: rc-service restart $rc_name failed",
|
||||||
|
"rc-update del \"$rc_name\" default",
|
||||||
|
} {
|
||||||
|
if !strings.Contains(script, want) {
|
||||||
|
t.Fatalf("openrc supervisor missing %q:\n%s", want, script)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestActivateScriptNoStartSkipsRestart(t *testing.T) {
|
||||||
|
script := activateScript(activateParams{
|
||||||
|
Stage: "/opt/rsmon-worker-src/rsmon-worker", Binary: "/usr/local/bin/rsmon-worker",
|
||||||
|
ConfigDir: "/etc/rsmon-worker", EnvFile: "/etc/rsmon-worker/worker.env", EnvTmp: "/tmp/e.env",
|
||||||
|
DataDir: "/var/lib/rsmon-worker", UnitTmp: "", UnitFile: "", UnitMode: "",
|
||||||
|
UnitName: "rsmon-worker.service", RCName: "rsmon-worker", Supervisor: "none", NoStart: "1",
|
||||||
|
})
|
||||||
|
if !strings.Contains(script, "[ \"$no_start\" -ne 1 ]") {
|
||||||
|
t.Fatalf("no-start gate missing:\n%s", script)
|
||||||
|
}
|
||||||
|
if !strings.Contains(script, "started=no") {
|
||||||
|
t.Fatalf("no-start summary missing:\n%s", script)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestActivateScriptNeverContainsSecrets(t *testing.T) {
|
||||||
|
const secret = "super-secret-token-value"
|
||||||
|
params := activateParams{
|
||||||
|
Stage: "/opt/rsmon-worker-src/rsmon-worker", Binary: "/usr/local/bin/rsmon-worker",
|
||||||
|
ConfigDir: "/etc/rsmon-worker", EnvFile: "/etc/rsmon-worker/worker.env", EnvTmp: "/tmp/e.env",
|
||||||
|
DataDir: "/var/lib/rsmon-worker", UnitTmp: "/tmp/u", UnitFile: "/etc/systemd/system/rsmon-worker.service",
|
||||||
|
UnitMode: "0644", UnitName: "rsmon-worker.service", RCName: "rsmon-worker",
|
||||||
|
Supervisor: "systemd", NoStart: "0",
|
||||||
|
}
|
||||||
|
script := activateScript(params)
|
||||||
|
if strings.Contains(script, secret) {
|
||||||
|
t.Fatalf("activate script contains a secret:\n%s", script)
|
||||||
|
}
|
||||||
|
// The env file is referenced by path; its contents (the secrets) are
|
||||||
|
// only sourced at runtime and never echoed.
|
||||||
|
for _, want := range []string{"RSMON_TOKEN=", secret} {
|
||||||
|
if strings.Contains(script, want) {
|
||||||
|
t.Fatalf("activate script must not embed env contents (%q):\n%s", want, script)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestOpenRCInit(t *testing.T) {
|
||||||
|
unit := openrcInitFor(resolvePaths(""))
|
||||||
|
for _, want := range []string{
|
||||||
|
"#!/sbin/openrc-run",
|
||||||
|
"name=rsmon-worker",
|
||||||
|
"description=RSMon distributed monitoring worker",
|
||||||
|
"command=/usr/local/bin/rsmon-worker",
|
||||||
|
"command_background=true",
|
||||||
|
"pidfile=/var/lib/rsmon-worker/worker.pid",
|
||||||
|
"output_log=/var/lib/rsmon-worker/worker.log",
|
||||||
|
"need net",
|
||||||
|
". /etc/rsmon-worker/worker.env",
|
||||||
|
"set -a",
|
||||||
|
"set +a",
|
||||||
|
"RSMON_WEBAPP_DATA_DIR=/var/lib/rsmon-worker/webapp",
|
||||||
|
} {
|
||||||
|
if !strings.Contains(unit, want) {
|
||||||
|
t.Fatalf("openrc init missing %q:\n%s", want, unit)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
named := openrcInitFor(resolvePaths("edge"))
|
||||||
|
for _, want := range []string{
|
||||||
|
"name=rsmon-worker-edge", "command=/usr/local/bin/rsmon-worker-edge",
|
||||||
|
"pidfile=/var/lib/rsmon-worker-edge/worker.pid", "/etc/rsmon-worker-edge/worker.env", "(edge)",
|
||||||
|
} {
|
||||||
|
if !strings.Contains(named, want) {
|
||||||
|
t.Fatalf("named openrc init missing %q:\n%s", want, named)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunningInitScript(t *testing.T) {
|
||||||
|
for _, want := range []string{"set -eu", "/run/systemd/system", "systemctl", "/run/openrc/softlevel", "rc-service", "echo none"} {
|
||||||
|
if !strings.Contains(runningInitScript, want) {
|
||||||
|
t.Fatalf("running-init script missing %q:\n%s", want, runningInitScript)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNormalizeSupervisor(t *testing.T) {
|
||||||
|
for _, in := range []string{"systemd", "openrc", "systemd\n", " openrc "} {
|
||||||
|
want := strings.TrimSpace(in)
|
||||||
|
if got := normalizeSupervisor(in); got != want {
|
||||||
|
t.Fatalf("normalizeSupervisor(%q) = %q, want %q", in, got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, in := range []string{"", "none", "sysvinit", " "} {
|
||||||
|
if got := normalizeSupervisor(in); got != "none" {
|
||||||
|
t.Fatalf("normalizeSupervisor(%q) = %q, want none", in, got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestUnitContent(t *testing.T) {
|
||||||
|
p := resolvePaths("")
|
||||||
|
systemd, file, mode := unitContent(sshinstall.InitSystemd, p)
|
||||||
|
if systemd == "" || file != "/etc/systemd/system/rsmon-worker.service" || mode != "0644" {
|
||||||
|
t.Fatalf("systemd unit content = %q, %q, %q", systemd, file, mode)
|
||||||
|
}
|
||||||
|
if !strings.Contains(systemd, "ExecStart=/usr/local/bin/rsmon-worker\n") {
|
||||||
|
t.Fatalf("systemd unit not the classic hardened unit:\n%s", systemd)
|
||||||
|
}
|
||||||
|
openrc, file, mode := unitContent(sshinstall.InitOpenRC, p)
|
||||||
|
if openrc == "" || file != "/etc/init.d/rsmon-worker" || mode != "0755" {
|
||||||
|
t.Fatalf("openrc unit content = %q, %q, %q", openrc, file, mode)
|
||||||
|
}
|
||||||
|
if content, file, mode := unitContent(sshinstall.InitUnknown, p); content != "" || file != "" || mode != "" {
|
||||||
|
t.Fatalf("unknown-init unit content = %q, %q, %q, want empty (no-service gate)", content, file, mode)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRenderEnvActivation(t *testing.T) {
|
||||||
|
t.Setenv("RSMON_URL", "")
|
||||||
|
t.Setenv("RSMON_TOKEN", "")
|
||||||
|
t.Setenv("WORKER_HOST", "")
|
||||||
|
t.Setenv("WORKER_PORT", "")
|
||||||
|
t.Setenv("WORKER_LOGIN", "")
|
||||||
|
t.Setenv("WORKER_PASSWORD", "")
|
||||||
|
|
||||||
|
data, err := (ActivationOptions{Activate: true, URL: "https://rsmon.ru", Token: "secret"}).renderEnv()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, want := range []string{"RSMON_URL=https://rsmon.ru\n", "RSMON_TOKEN=secret\n", "WORKER_HOST=127.0.0.1\n", "WORKER_PORT=27401\n"} {
|
||||||
|
if !strings.Contains(string(data), want) {
|
||||||
|
t.Fatalf("rendered env missing %q:\n%s", want, data)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// PUBLIC_URL canonicalization is reused from the classic installer.
|
||||||
|
data, err = (ActivationOptions{
|
||||||
|
Activate: true, URL: "https://rsmon.ru", Token: "secret",
|
||||||
|
PublicURL: "https://worker.example.com",
|
||||||
|
}).renderEnv()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !strings.Contains(string(data), "PUBLIC_URL=https://worker.example.com\n") {
|
||||||
|
t.Fatalf("rendered env missing PUBLIC_URL:\n%s", data)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRenderEnvActivationRejectsMixedBasicAuth(t *testing.T) {
|
||||||
|
t.Setenv("RSMON_TOKEN", "")
|
||||||
|
if _, err := (ActivationOptions{
|
||||||
|
Activate: true, URL: "https://rsmon.ru", Token: "secret",
|
||||||
|
Login: "admin",
|
||||||
|
}).renderEnv(); err == nil {
|
||||||
|
t.Fatal("login-only basic auth accepted")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRenderEnvActivationRequiresToken(t *testing.T) {
|
||||||
|
if _, err := (ActivationOptions{Activate: true, URL: "https://rsmon.ru"}).renderEnv(); err == nil {
|
||||||
|
t.Fatal("activation without a token accepted")
|
||||||
|
}
|
||||||
|
if _, err := (ActivationOptions{Activate: true, Token: "x", URL: "not-a-url"}).renderEnv(); err == nil {
|
||||||
|
t.Fatal("activation with a malformed URL accepted")
|
||||||
|
}
|
||||||
|
if _, err := (ActivationOptions{Activate: true, Token: "x", URL: "https://rsmon.ru", Name: "Bad_Name"}).renderEnv(); err == nil {
|
||||||
|
t.Fatal("activation with an invalid instance name accepted")
|
||||||
|
}
|
||||||
|
if _, err := (ActivationOptions{Activate: true, Token: "x", URL: "https://rsmon.ru", Name: "edge"}).renderEnv(); err == nil {
|
||||||
|
t.Fatal("named activation without WORKER_PORT accepted")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestRenderEnvActivationReadsEnvFileOnce proves the env file is read a
|
||||||
|
// single time and validated from the in-memory bytes: swapping the file
|
||||||
|
// after the read cannot smuggle a different value into the render, and a
|
||||||
|
// malformed file fails on the read bytes.
|
||||||
|
func TestRenderEnvActivationReadsEnvFileOnce(t *testing.T) {
|
||||||
|
t.Setenv("RSMON_TOKEN", "")
|
||||||
|
path := filepath.Join(t.TempDir(), "worker.env")
|
||||||
|
if err := os.WriteFile(path, []byte("RSMON_URL=https://rsmon.ru\nRSMON_TOKEN=file-token\nWORKER_PORT=28888\n"), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
data, err := (ActivationOptions{Activate: true, EnvFile: path}).renderEnv()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !strings.Contains(string(data), "RSMON_TOKEN=file-token\n") || !strings.Contains(string(data), "WORKER_PORT=28888\n") {
|
||||||
|
t.Fatalf("env file values not rendered:\n%s", data)
|
||||||
|
}
|
||||||
|
// A malformed file must be rejected from the same read.
|
||||||
|
if err := os.WriteFile(path, []byte("RSMON_URL=https://rsmon.ru\nRSMON_TOKEN=has space\n"), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := (ActivationOptions{Activate: true, EnvFile: path}).renderEnv(); err == nil {
|
||||||
|
t.Fatal("malformed env file accepted")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestNormalizeSourceOptionsActivationRendersOnce verifies the rendered
|
||||||
|
// env is computed during normalization and reused, so a later call cannot
|
||||||
|
// re-read a changed env file.
|
||||||
|
func TestNormalizeSourceOptionsActivationRendersOnce(t *testing.T) {
|
||||||
|
t.Setenv("RSMON_TOKEN", "")
|
||||||
|
path := filepath.Join(t.TempDir(), "worker.env")
|
||||||
|
if err := os.WriteFile(path, []byte("RSMON_URL=https://rsmon.ru\nRSMON_TOKEN=once-token\n"), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
base := SourceInstallOptions{
|
||||||
|
SSHOptions: SSHOptions{Host: "h", User: "u"},
|
||||||
|
Activation: ActivationOptions{Activate: true, EnvFile: path},
|
||||||
|
}
|
||||||
|
norm, err := normalizeSourceOptions(base)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !strings.Contains(string(norm.activationEnv), "RSMON_TOKEN=once-token\n") {
|
||||||
|
t.Fatalf("activation env not cached during normalization: %q", norm.activationEnv)
|
||||||
|
}
|
||||||
|
// Even after the file changes, the cached render is authoritative.
|
||||||
|
if err := os.WriteFile(path, []byte("RSMON_URL=https://evil.test\nRSMON_TOKEN=evil\n"), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if strings.Contains(string(norm.activationEnv), "evil") {
|
||||||
|
t.Fatalf("changed env file leaked into the cached render: %q", norm.activationEnv)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNormalizeSourceOptionsActivation(t *testing.T) {
|
||||||
|
t.Setenv("RSMON_TOKEN", "")
|
||||||
|
base := SourceInstallOptions{SSHOptions: SSHOptions{Host: "h", User: "u"}}
|
||||||
|
if _, err := normalizeSourceOptions(base); err != nil {
|
||||||
|
t.Fatalf("staging-only options must stay valid: %v", err)
|
||||||
|
}
|
||||||
|
act := base
|
||||||
|
act.Activation = ActivationOptions{Activate: true, URL: "https://rsmon.ru"}
|
||||||
|
if _, err := normalizeSourceOptions(act); err == nil || !strings.Contains(err.Error(), "RSMON_TOKEN") {
|
||||||
|
t.Fatalf("activation without a token error = %v", err)
|
||||||
|
}
|
||||||
|
ok := base
|
||||||
|
ok.Activation = ActivationOptions{Activate: true, URL: "https://rsmon.ru", Token: "secret"}
|
||||||
|
if _, err := normalizeSourceOptions(ok); err != nil {
|
||||||
|
t.Fatalf("valid activation rejected: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestValidateRenderedEnv(t *testing.T) {
|
||||||
|
if err := validateRenderedEnv([]byte("RSMON_URL=https://rsmon.ru\nRSMON_TOKEN=secret\n")); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := validateRenderedEnv([]byte("RSMON_TOKEN=secret\n")); err == nil {
|
||||||
|
t.Fatal("env without RSMON_URL accepted")
|
||||||
|
}
|
||||||
|
if err := validateRenderedEnv([]byte("RSMON_URL=https://rsmon.ru\nRSMON_TOKEN=secret value\n")); err == nil {
|
||||||
|
t.Fatal("env with whitespace accepted")
|
||||||
|
}
|
||||||
|
}
|
||||||
603
internal/installer/sourceinstall.go
Обычный файл
603
internal/installer/sourceinstall.go
Обычный файл
@@ -0,0 +1,603 @@
|
|||||||
|
package installer
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"net"
|
||||||
|
"path/filepath"
|
||||||
|
"regexp"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"golang.org/x/crypto/ssh"
|
||||||
|
|
||||||
|
"rocketgit.ru/rsmon/worker/internal/sshinstall"
|
||||||
|
)
|
||||||
|
|
||||||
|
// SourceInstallOptions drives the remote source installation (work
|
||||||
|
// package 3 of docs/source-installation.md). It reuses the deploy
|
||||||
|
// command's SSHOptions for authentication and host-key verification and
|
||||||
|
// adds the source-build knobs. It never carries a worker token or
|
||||||
|
// control-plane credential: the built worker is staged, not configured
|
||||||
|
// or started, so no secret is ever sent to the remote host.
|
||||||
|
type SourceInstallOptions struct {
|
||||||
|
SSHOptions
|
||||||
|
|
||||||
|
// Repo is the public worker repository to clone or update. Empty
|
||||||
|
// uses the sshinstall default. Only https URLs without userinfo are
|
||||||
|
// accepted.
|
||||||
|
Repo string
|
||||||
|
// Branch pins the branch to build. Empty resolves the remote's
|
||||||
|
// default branch (the public repo currently publishes "master");
|
||||||
|
// the resolved branch and commit are recorded in the build dir.
|
||||||
|
Branch string
|
||||||
|
// GoVersion defaults to the pinned sshinstall toolchain (1.26.0).
|
||||||
|
// Non-default versions have no baked checksum yet and are rejected.
|
||||||
|
GoVersion string
|
||||||
|
// GoArch optionally pins the Go download archive suffix (e.g.
|
||||||
|
// "amd64"); empty derives it from the remote `uname -m`.
|
||||||
|
GoArch string
|
||||||
|
// BuildDir is the remote clone/build directory.
|
||||||
|
BuildDir string
|
||||||
|
// GoModuleProxy overrides GOPROXY for the remote build.
|
||||||
|
GoModuleProxy string
|
||||||
|
// ToolchainDir is where the verified Go toolchain is installed.
|
||||||
|
// It must be an absolute path ending in /go (default
|
||||||
|
// /usr/local/go). Replacement is atomic: the new toolchain is
|
||||||
|
// downloaded, verified, and staged before the prior one is moved
|
||||||
|
// aside, and the prior one is restored if the swap fails.
|
||||||
|
ToolchainDir string
|
||||||
|
// StageBinary is where the built worker binary is written. It must
|
||||||
|
// be absolute and defaults to <BuildDir>/rsmon-worker. The running
|
||||||
|
// service and its config are NOT touched until Activation runs.
|
||||||
|
StageBinary string
|
||||||
|
// SessionTimeout bounds each remote command. 0 uses the default
|
||||||
|
// (30 minutes); the build step can legitimately run for minutes.
|
||||||
|
SessionTimeout time.Duration
|
||||||
|
// Activation drives work package 4: after the staging build, the
|
||||||
|
// staged binary, validated environment, data dir, and the detected
|
||||||
|
// init's service definition are installed atomically and the worker
|
||||||
|
// is started and verified (process + /healthz). Any failure rolls
|
||||||
|
// back to the prior working install. Empty keeps SourceInstall at the
|
||||||
|
// staging boundary and touches no service configuration.
|
||||||
|
Activation ActivationOptions
|
||||||
|
|
||||||
|
// activationEnv is the once-rendered activation environment, computed
|
||||||
|
// during option normalization so the env file is read and rendered
|
||||||
|
// exactly once per run (no TOCTOU between preflight and activation).
|
||||||
|
activationEnv []byte
|
||||||
|
}
|
||||||
|
|
||||||
|
// SourceInstallResult is what a source installation resolved to. The
|
||||||
|
// resolved branch and commit are recorded on the remote host in
|
||||||
|
// RecordFile, and the built binary is left at StageBinary for the next
|
||||||
|
// (service-activation) work package to install atomically.
|
||||||
|
type SourceInstallResult struct {
|
||||||
|
Detection sshinstall.Detection
|
||||||
|
Plan sshinstall.SourcePlan
|
||||||
|
GoArch string // resolved archive suffix, e.g. "linux-amd64"
|
||||||
|
ToolchainDir string
|
||||||
|
ResolvedBranch string
|
||||||
|
ResolvedCommit string
|
||||||
|
RecordFile string
|
||||||
|
StageBinary string
|
||||||
|
// Activation is set when the staged binary was atomically installed
|
||||||
|
// and the worker started/verified. It records the installed layout
|
||||||
|
// and the supervisor used. Nil when Activation was not requested.
|
||||||
|
Activation *ActivationResult
|
||||||
|
}
|
||||||
|
|
||||||
|
// defaultToolchainDir is the standard Go installation prefix.
|
||||||
|
const defaultToolchainDir = "/usr/local/go"
|
||||||
|
|
||||||
|
// defaultSessionTimeout bounds each remote command when the operator
|
||||||
|
// does not configure one. The staging build and first-time module
|
||||||
|
// downloads can run for minutes, so this is generous.
|
||||||
|
const defaultSessionTimeout = 30 * time.Minute
|
||||||
|
|
||||||
|
// commitRecordName is the file (inside BuildDir) that records the
|
||||||
|
// resolved branch and commit the build was produced from.
|
||||||
|
const commitRecordName = "rsmon-worker.commit"
|
||||||
|
|
||||||
|
var (
|
||||||
|
commitPattern = regexp.MustCompile(`^[0-9a-f]{40}$`)
|
||||||
|
branchNamePattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._/-]*$`)
|
||||||
|
)
|
||||||
|
|
||||||
|
// sourceExecutor bundles the SSH client, privilege options, and per
|
||||||
|
// command timeout used by every remote source-install step.
|
||||||
|
type sourceExecutor struct {
|
||||||
|
client *ssh.Client
|
||||||
|
ssh SSHOptions
|
||||||
|
timeout time.Duration
|
||||||
|
}
|
||||||
|
|
||||||
|
// runPrivileged executes a command through the SSHOptions privilege path
|
||||||
|
// (root, passwordless sudo, or sudo -S) and returns bounded stdout.
|
||||||
|
func (e *sourceExecutor) runPrivileged(command string) ([]byte, error) {
|
||||||
|
cmd, stdin := sudoWrap(e.ssh.User, e.ssh.SudoPassword, command)
|
||||||
|
return runRemoteOutput(e.client, cmd, stdin, e.timeout)
|
||||||
|
}
|
||||||
|
|
||||||
|
// runPlain executes a command as the SSH user and returns bounded
|
||||||
|
// stdout.
|
||||||
|
func (e *sourceExecutor) runPlain(command string) ([]byte, error) {
|
||||||
|
return runRemoteOutput(e.client, command, nil, e.timeout)
|
||||||
|
}
|
||||||
|
|
||||||
|
// fileProber builds the sshinstall.FileProber used for init-system
|
||||||
|
// detection over a live SSH session.
|
||||||
|
func (e *sourceExecutor) fileProber() sshinstall.FileProber {
|
||||||
|
return func(paths ...string) map[string]bool {
|
||||||
|
quoted := make([]string, len(paths))
|
||||||
|
for i, p := range paths {
|
||||||
|
quoted[i] = shellQuote(p)
|
||||||
|
}
|
||||||
|
expr := "for p in " + strings.Join(quoted, " ") + "; do [ -e \"$p\" ] && printf '%s\\n' \"$p\"; done; true"
|
||||||
|
out, err := e.runPlain(expr)
|
||||||
|
if err != nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
present := make(map[string]bool, len(paths))
|
||||||
|
for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") {
|
||||||
|
if line = strings.TrimSpace(line); line != "" {
|
||||||
|
present[line] = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return present
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// SourceInstall executes the source-install flow over SSH: it reuses the
|
||||||
|
// deploy command's SSH authentication and host-key verification, detects
|
||||||
|
// the remote host, plans the pinned toolchain and package prerequisites,
|
||||||
|
// then installs packages, downloads and verifies the Go toolchain,
|
||||||
|
// clones/updates the public repository (verifying an existing checkout's
|
||||||
|
// origin matches the configured repository), checks out the resolved
|
||||||
|
// branch, builds the worker to a staging path, and only then records the
|
||||||
|
// resolved branch and commit.
|
||||||
|
//
|
||||||
|
// When Activation.Activate is set, the staged binary is then atomically
|
||||||
|
// installed together with the validated environment, data directory, and
|
||||||
|
// the detected init's service definition, and the worker is started and
|
||||||
|
// verified (process + /healthz); any activation/start/health failure
|
||||||
|
// rolls back to the prior working install. Without activation the
|
||||||
|
// running service, its configuration, and its data directory are
|
||||||
|
// deliberately untouched (the staging boundary of work package 3).
|
||||||
|
//
|
||||||
|
// Every remote step runs with the same privilege path as `deploy` (root,
|
||||||
|
// passwordless sudo, or sudo -S), every interpolated value is
|
||||||
|
// single-quoted, every step script fails closed (`set -eu` or explicit
|
||||||
|
// `&&`/retry), and every failure returns a bounded, actionable error.
|
||||||
|
// Each remote command is capped by SessionTimeout and its stdout is
|
||||||
|
// size-bounded.
|
||||||
|
func SourceInstall(opts SourceInstallOptions) (*SourceInstallResult, error) {
|
||||||
|
opts, err := normalizeSourceOptions(opts)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
auth, err := sshAuth(opts.SSHOptions)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
hostKey, err := hostKeyCallback(opts.SSHOptions)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
client, err := ssh.Dial("tcp", net.JoinHostPort(opts.Host, strconv.Itoa(opts.Port)), &ssh.ClientConfig{
|
||||||
|
User: opts.User,
|
||||||
|
Auth: auth,
|
||||||
|
HostKeyCallback: hostKey,
|
||||||
|
Timeout: 15 * time.Second,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("connect to %s: %w", opts.Host, err)
|
||||||
|
}
|
||||||
|
defer client.Close() //nolint:errcheck
|
||||||
|
|
||||||
|
executor := &sourceExecutor{client: client, ssh: opts.SSHOptions, timeout: opts.SessionTimeout}
|
||||||
|
|
||||||
|
osRelease, err := executor.runPlain("cat /etc/os-release")
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("read remote /etc/os-release: %w", err)
|
||||||
|
}
|
||||||
|
uname, err := executor.runPlain("uname -m")
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("read remote machine architecture: %w", err)
|
||||||
|
}
|
||||||
|
detection := sshinstall.Detect(string(osRelease), executor.fileProber())
|
||||||
|
plan, err := sshinstall.PlanSource(detection, sshinstall.SourceOptions{
|
||||||
|
Repo: opts.Repo,
|
||||||
|
Branch: opts.Branch,
|
||||||
|
GoVersion: opts.GoVersion,
|
||||||
|
GoArch: opts.GoArch,
|
||||||
|
UnameM: strings.TrimSpace(string(uname)),
|
||||||
|
BuildDir: opts.BuildDir,
|
||||||
|
GoModuleProxy: opts.GoModuleProxy,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
stage := opts.StageBinary
|
||||||
|
if stage == "" {
|
||||||
|
stage = filepath.Join(plan.BuildDir, "rsmon-worker")
|
||||||
|
}
|
||||||
|
result := &SourceInstallResult{
|
||||||
|
Detection: detection,
|
||||||
|
Plan: plan,
|
||||||
|
GoArch: plan.Toolchain.Arch,
|
||||||
|
ToolchainDir: opts.ToolchainDir,
|
||||||
|
StageBinary: stage,
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(plan.Packages) > 0 {
|
||||||
|
script := "sh -c " + shellQuote(packageScript(detection.PackageManager, plan.Packages))
|
||||||
|
if _, err := executor.runPrivileged(script); err != nil {
|
||||||
|
return nil, fmt.Errorf("install prerequisites via %s: %w", detection.PackageManager, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
script := "sh -c " + shellQuote(toolchainScript(plan.Toolchain, opts.ToolchainDir))
|
||||||
|
if _, err := executor.runPrivileged(script); err != nil {
|
||||||
|
return nil, fmt.Errorf("install Go %s toolchain: %w", plan.Toolchain.Version, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Clone or update. An existing checkout must point at the configured
|
||||||
|
// repository, or the install fails before fetching or building.
|
||||||
|
script = "sh -c " + shellQuote(cloneUpdateScript(plan.Repo, plan.BuildDir))
|
||||||
|
if _, err := executor.runPrivileged(script); err != nil {
|
||||||
|
return nil, fmt.Errorf("clone/update source repository: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
out, err := executor.runPrivileged("sh -c " + shellQuote(resolveBranchScript(plan.BuildDir)))
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("resolve remote default branch: %w", err)
|
||||||
|
}
|
||||||
|
branch, err := parseResolvedBranch(string(out))
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if plan.Branch != "" && plan.Branch != branch {
|
||||||
|
ref := "refs/remotes/origin/" + plan.Branch
|
||||||
|
if _, err := executor.runPrivileged("sh -c " + shellQuote(refExistsScript(plan.BuildDir, ref))); err != nil {
|
||||||
|
return nil, fmt.Errorf("branch %q does not exist on the remote repository: %w", plan.Branch, err)
|
||||||
|
}
|
||||||
|
branch = plan.Branch
|
||||||
|
}
|
||||||
|
result.ResolvedBranch = branch
|
||||||
|
|
||||||
|
// Checkout fails closed: a dirty tree or missing branch aborts before
|
||||||
|
// the build, so the previous staging binary is left untouched.
|
||||||
|
out, err = executor.runPrivileged("sh -c " + shellQuote(checkoutScript(plan.BuildDir, branch)))
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("check out branch %q: %w", branch, err)
|
||||||
|
}
|
||||||
|
commit, err := parseResolvedCommit(string(out))
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("resolve commit: %w", err)
|
||||||
|
}
|
||||||
|
result.ResolvedCommit = commit
|
||||||
|
|
||||||
|
// Build with the repository's own flags (same -ldflags shape the
|
||||||
|
// worker Makefile uses), resolving COMMIT from the checkout and
|
||||||
|
// BUILD_DATE at install time. The binary is built to a temp path,
|
||||||
|
// verified, and atomically swapped into the staging path; a failed
|
||||||
|
// build leaves the previous staging binary in place.
|
||||||
|
buildDate := time.Now().UTC().Format("2006-01-02T15:04:05Z")
|
||||||
|
ldflags := fmt.Sprintf("-s -w -X main.version=dev -X main.commit=%s -X main.buildDate=%s", commit[:12], buildDate)
|
||||||
|
script = "sh -c " + shellQuote(buildScript(opts.ToolchainDir, plan.BuildDir, plan.GoModuleProxy, stage, ldflags))
|
||||||
|
if _, err := executor.runPrivileged(script); err != nil {
|
||||||
|
return nil, fmt.Errorf("build worker binary: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// The commit record is written only after a successful build, so the
|
||||||
|
// record and the staged binary always correspond to the same commit.
|
||||||
|
result.RecordFile = filepath.Join(plan.BuildDir, commitRecordName)
|
||||||
|
if _, err := executor.runPrivileged("sh -c " + shellQuote(commitRecordScript(plan.BuildDir, branch, commit))); err != nil {
|
||||||
|
return nil, fmt.Errorf("record resolved commit: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Work package 4: atomically install the staged build and activate
|
||||||
|
// the worker. On any failure the remote script restores the previous
|
||||||
|
// working install and this step returns a bounded error.
|
||||||
|
if opts.Activation.Activate {
|
||||||
|
if err := executor.activateWorker(opts, result); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return result, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// normalizeSourceOptions validates operator input before any remote
|
||||||
|
// connection or mutation. Every value that later reaches a remote shell
|
||||||
|
// is constrained here.
|
||||||
|
func normalizeSourceOptions(o SourceInstallOptions) (SourceInstallOptions, error) {
|
||||||
|
if o.Host == "" || o.User == "" {
|
||||||
|
return o, errors.New("--host and --user are required for source install")
|
||||||
|
}
|
||||||
|
if o.Port == 0 {
|
||||||
|
o.Port = 22
|
||||||
|
}
|
||||||
|
if o.Port < 1 || o.Port > 65535 {
|
||||||
|
return o, errors.New("SSH port must be between 1 and 65535")
|
||||||
|
}
|
||||||
|
if o.Branch != "" && !validBranchName(o.Branch) {
|
||||||
|
return o, fmt.Errorf("invalid branch %q: only A-Za-z0-9, dots, underscores, slashes, and hyphens are allowed", o.Branch)
|
||||||
|
}
|
||||||
|
if o.GoVersion != "" && !sshinstall.ValidGoVersion(o.GoVersion) {
|
||||||
|
return o, fmt.Errorf("invalid Go version %q: only digits, letters, dots, dashes, and underscores are allowed", o.GoVersion)
|
||||||
|
}
|
||||||
|
if o.GoArch != "" && !sshinstall.ValidGoArch(o.GoArch) {
|
||||||
|
return o, fmt.Errorf("invalid Go architecture %q: only letters, digits, dashes, and underscores are allowed", o.GoArch)
|
||||||
|
}
|
||||||
|
if err := sshinstall.ValidateRepoURL(o.Repo); err != nil {
|
||||||
|
return o, fmt.Errorf("invalid repository: %w", err)
|
||||||
|
}
|
||||||
|
if o.ToolchainDir == "" {
|
||||||
|
o.ToolchainDir = defaultToolchainDir
|
||||||
|
}
|
||||||
|
if !strings.HasPrefix(o.ToolchainDir, "/") || filepath.Base(o.ToolchainDir) != "go" {
|
||||||
|
return o, fmt.Errorf("toolchain directory must be an absolute path ending in /go, got %q", o.ToolchainDir)
|
||||||
|
}
|
||||||
|
if o.StageBinary != "" && !strings.HasPrefix(o.StageBinary, "/") {
|
||||||
|
return o, fmt.Errorf("staging binary path must be absolute, got %q", o.StageBinary)
|
||||||
|
}
|
||||||
|
if o.SessionTimeout <= 0 {
|
||||||
|
o.SessionTimeout = defaultSessionTimeout
|
||||||
|
}
|
||||||
|
if o.Activation.Activate {
|
||||||
|
// Render (and validate) the activation environment exactly once
|
||||||
|
// here. The rendered bytes are reused at activation time, so the
|
||||||
|
// env file is read a single time and cannot change between the
|
||||||
|
// preflight and the remote install (env-file TOCTOU).
|
||||||
|
env, err := o.Activation.renderEnv()
|
||||||
|
if err != nil {
|
||||||
|
return o, fmt.Errorf("activation: %w", err)
|
||||||
|
}
|
||||||
|
if err := validateRenderedEnv(env); err != nil {
|
||||||
|
return o, fmt.Errorf("activation: %w", err)
|
||||||
|
}
|
||||||
|
o.activationEnv = env
|
||||||
|
}
|
||||||
|
return o, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// validBranchName reports whether a branch is a safe git branch name
|
||||||
|
// that can be interpolated into remote commands. The charset check is
|
||||||
|
// the command-injection boundary; the extra rules reject git-invalid or
|
||||||
|
// ambiguous refname patterns.
|
||||||
|
func validBranchName(branch string) bool {
|
||||||
|
if !branchNamePattern.MatchString(branch) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
if strings.HasPrefix(branch, "-") || strings.HasPrefix(branch, "/") ||
|
||||||
|
strings.Contains(branch, "..") || strings.Contains(branch, "@{") ||
|
||||||
|
strings.Contains(branch, "//") || strings.HasSuffix(branch, ".") ||
|
||||||
|
strings.HasSuffix(branch, "/") {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
// parseResolvedBranch turns the `git symbolic-ref` output
|
||||||
|
// ("origin/master\n") into the short branch name, validating it so
|
||||||
|
// remote-controlled output can never inject a command.
|
||||||
|
func parseResolvedBranch(raw string) (string, error) {
|
||||||
|
branch := strings.TrimSpace(raw)
|
||||||
|
branch = strings.TrimPrefix(branch, "origin/")
|
||||||
|
if !validBranchName(branch) {
|
||||||
|
return "", fmt.Errorf("remote reported an invalid default branch %q", strings.TrimSpace(raw))
|
||||||
|
}
|
||||||
|
return branch, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// parseResolvedCommit extracts the 40-hex commit from `git rev-parse
|
||||||
|
// HEAD` output, taking the last whitespace-separated token so unrelated
|
||||||
|
// stdout cannot satisfy the parse.
|
||||||
|
func parseResolvedCommit(raw string) (string, error) {
|
||||||
|
fields := strings.Fields(strings.TrimSpace(raw))
|
||||||
|
if len(fields) == 0 {
|
||||||
|
return "", errors.New("remote reported no commit")
|
||||||
|
}
|
||||||
|
commit := fields[len(fields)-1]
|
||||||
|
if !commitPattern.MatchString(commit) {
|
||||||
|
return "", fmt.Errorf("remote reported an invalid resolved commit %q", commit)
|
||||||
|
}
|
||||||
|
return commit, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// quoteList renders each item as a single shell-quoted word.
|
||||||
|
func quoteList(items []string) string {
|
||||||
|
quoted := make([]string, len(items))
|
||||||
|
for i, item := range items {
|
||||||
|
quoted[i] = shellQuote(item)
|
||||||
|
}
|
||||||
|
return strings.Join(quoted, " ")
|
||||||
|
}
|
||||||
|
|
||||||
|
// packageScript installs the minimal build prerequisites with the
|
||||||
|
// distro's package manager. It is idempotent on every supported manager
|
||||||
|
// and never installs a C compiler (the worker builds with CGO disabled).
|
||||||
|
func packageScript(pkg sshinstall.PackageManager, pkgs []string) string {
|
||||||
|
quoted := quoteList(pkgs)
|
||||||
|
switch pkg {
|
||||||
|
case sshinstall.PkgApk:
|
||||||
|
return "apk add --no-cache " + quoted
|
||||||
|
case sshinstall.PkgApt:
|
||||||
|
// Ubuntu/Debian need a fresh package index before installing.
|
||||||
|
return "export DEBIAN_FRONTEND=noninteractive\napt-get update\napt-get install -y --no-install-recommends " + quoted
|
||||||
|
case sshinstall.PkgPacman:
|
||||||
|
return "pacman -Sy --noconfirm --needed " + quoted
|
||||||
|
case sshinstall.PkgDnf:
|
||||||
|
return "dnf install -y --setopt=install_weak_deps=False " + quoted
|
||||||
|
case sshinstall.PkgYum:
|
||||||
|
return "yum install -y " + quoted
|
||||||
|
default:
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// toolchainScript downloads the pinned Go toolchain, verifies its
|
||||||
|
// published SHA-256 before extraction, stages the extract on the same
|
||||||
|
// filesystem as the target, verifies the staged toolchain reports the
|
||||||
|
// target version, and only then swaps it into place. The prior toolchain
|
||||||
|
// (when present) is moved to a sibling backup first and is restored if
|
||||||
|
// the swap fails, so a failed download/verify/extract/swap always leaves
|
||||||
|
// the prior Go untouched. A present toolchain already reporting the
|
||||||
|
// target version is reused (idempotent rerun). Temp and staging
|
||||||
|
// directories are removed on success and failure.
|
||||||
|
func toolchainScript(tc sshinstall.Toolchain, toolchainDir string) string {
|
||||||
|
parent := filepath.Dir(toolchainDir)
|
||||||
|
want := "go" + tc.Version
|
||||||
|
var b strings.Builder
|
||||||
|
b.WriteString("set -eu\n")
|
||||||
|
b.WriteString("parent=" + shellQuote(parent) + "\n")
|
||||||
|
b.WriteString("toolchain=" + shellQuote(toolchainDir) + "\n")
|
||||||
|
b.WriteString("want=" + shellQuote(want) + "\n")
|
||||||
|
b.WriteString("if [ -x \"$toolchain/bin/go\" ]; then\n")
|
||||||
|
b.WriteString(" have=\"$($toolchain/bin/go version 2>/dev/null | awk '{print $3}')\"\n")
|
||||||
|
b.WriteString(" if [ \"$have\" = \"$want\" ]; then\n")
|
||||||
|
b.WriteString(" echo \"go toolchain already present: $have\"\n")
|
||||||
|
b.WriteString(" exit 0\n")
|
||||||
|
b.WriteString(" fi\n")
|
||||||
|
b.WriteString("fi\n")
|
||||||
|
b.WriteString("work=\"$(mktemp -d /tmp/rsmon-toolchain-XXXXXX)\"\n")
|
||||||
|
b.WriteString("staging=\"$(mktemp -d \"$parent/.go-staging-XXXXXX\")\"\n")
|
||||||
|
b.WriteString("trap 'rm -rf \"$work\" \"$staging\"' EXIT HUP INT TERM\n")
|
||||||
|
b.WriteString("archive=\"$work/go" + tc.Version + "." + tc.Arch + ".tar.gz\"\n")
|
||||||
|
b.WriteString("curl -fsSL --retry 3 --retry-delay 2 -o \"$archive\" " + shellQuote(tc.URL) + "\n")
|
||||||
|
b.WriteString("printf '%s %s\\n' " + shellQuote(tc.SHA256) + " \"$archive\" | sha256sum -c -\n")
|
||||||
|
b.WriteString("tar -C \"$staging\" -xzf \"$archive\"\n")
|
||||||
|
b.WriteString("staged=\"$($staging/go/bin/go version | awk '{print $3}')\"\n")
|
||||||
|
b.WriteString("if [ \"$staged\" != \"$want\" ]; then\n")
|
||||||
|
b.WriteString(" printf 'staged toolchain failed verification: %s\\n' \"$staged\" >&2\n")
|
||||||
|
b.WriteString(" exit 1\n")
|
||||||
|
b.WriteString("fi\n")
|
||||||
|
// Swap atomically on the same filesystem, preserving the prior
|
||||||
|
// toolchain in a sibling backup with rollback on failure.
|
||||||
|
b.WriteString("backup=\"\"\n")
|
||||||
|
b.WriteString("if [ -e \"$toolchain\" ]; then\n")
|
||||||
|
b.WriteString(" backup=\"$parent/.go-backup\"\n")
|
||||||
|
b.WriteString(" rm -rf \"$backup\"\n")
|
||||||
|
b.WriteString(" mv \"$toolchain\" \"$backup\"\n")
|
||||||
|
b.WriteString("fi\n")
|
||||||
|
b.WriteString("if ! mv \"$staging/go\" \"$toolchain\"; then\n")
|
||||||
|
b.WriteString(" if [ -n \"$backup\" ]; then\n")
|
||||||
|
b.WriteString(" mv \"$backup\" \"$toolchain\"\n")
|
||||||
|
b.WriteString(" fi\n")
|
||||||
|
b.WriteString(" exit 1\n")
|
||||||
|
b.WriteString("fi\n")
|
||||||
|
b.WriteString("if [ -n \"$backup\" ]; then\n")
|
||||||
|
b.WriteString(" rm -rf \"$backup\"\n")
|
||||||
|
b.WriteString("fi\n")
|
||||||
|
b.WriteString("\"$toolchain/bin/go\" version\n")
|
||||||
|
return b.String()
|
||||||
|
}
|
||||||
|
|
||||||
|
// cloneUpdateScript clones the repository when missing and otherwise
|
||||||
|
// fetches the latest refs, so a rerun updates in place. Before fetching
|
||||||
|
// an existing checkout, it verifies the configured repository matches
|
||||||
|
// the checkout's `remote.origin.url` exactly, so the installer can never
|
||||||
|
// fetch or build an unconfigured repository. The clone/fetch is retried
|
||||||
|
// up to three times (2s apart) because real repositories can be
|
||||||
|
// transiently unreachable (DNS, TLS, or proxy hiccups). The script fails
|
||||||
|
// closed: any exhausted retry exits non-zero.
|
||||||
|
func cloneUpdateScript(repo, buildDir string) string {
|
||||||
|
return "set -u\n" +
|
||||||
|
"repo=" + shellQuote(repo) + "\n" +
|
||||||
|
"dir=" + shellQuote(buildDir) + "\n" +
|
||||||
|
"if [ ! -d \"$dir/.git\" ]; then\n" +
|
||||||
|
" attempt=0\n" +
|
||||||
|
" while [ \"$attempt\" -lt 3 ]; do\n" +
|
||||||
|
" if git clone \"$repo\" \"$dir\"; then\n" +
|
||||||
|
" exit 0\n" +
|
||||||
|
" fi\n" +
|
||||||
|
" attempt=$((attempt + 1))\n" +
|
||||||
|
" sleep 2\n" +
|
||||||
|
" done\n" +
|
||||||
|
" exit 1\n" +
|
||||||
|
"fi\n" +
|
||||||
|
"origin=\"$(git -C \"$dir\" config --get remote.origin.url || true)\"\n" +
|
||||||
|
"if [ \"$origin\" != \"$repo\" ]; then\n" +
|
||||||
|
" printf 'existing checkout origin does not match configured repository\\nconfigured: %s\\nfound: %s\\n' \"$repo\" \"$origin\" >&2\n" +
|
||||||
|
" exit 1\n" +
|
||||||
|
"fi\n" +
|
||||||
|
"attempt=0\n" +
|
||||||
|
"while [ \"$attempt\" -lt 3 ]; do\n" +
|
||||||
|
" if git -C \"$dir\" fetch --prune origin; then\n" +
|
||||||
|
" exit 0\n" +
|
||||||
|
" fi\n" +
|
||||||
|
" attempt=$((attempt + 1))\n" +
|
||||||
|
" sleep 2\n" +
|
||||||
|
"done\n" +
|
||||||
|
"exit 1\n"
|
||||||
|
}
|
||||||
|
|
||||||
|
// resolveBranchScript prints the remote's default branch short name
|
||||||
|
// (with an "origin/" prefix) via origin/HEAD. It fails closed so a
|
||||||
|
// set-head failure aborts rather than resolving a stale default.
|
||||||
|
func resolveBranchScript(buildDir string) string {
|
||||||
|
return "set -eu\n" +
|
||||||
|
"git -C " + shellQuote(buildDir) + " remote set-head origin --auto >/dev/null\n" +
|
||||||
|
"git -C " + shellQuote(buildDir) + " symbolic-ref --short refs/remotes/origin/HEAD\n"
|
||||||
|
}
|
||||||
|
|
||||||
|
// refExistsScript verifies a remote-tracking ref exists (exit 0) without
|
||||||
|
// emitting output.
|
||||||
|
func refExistsScript(buildDir, ref string) string {
|
||||||
|
return "git -C " + shellQuote(buildDir) + " show-ref --verify --quiet " + shellQuote(ref)
|
||||||
|
}
|
||||||
|
|
||||||
|
// checkoutScript moves the local branch to the resolved remote branch
|
||||||
|
// and prints the resolved commit. It fails closed (`set -eu`): before the
|
||||||
|
// destructive `checkout -B` (which would silently discard local changes)
|
||||||
|
// it refuses when the tracked working tree is dirty, so a checkout
|
||||||
|
// failure can never be masked by a stale `rev-parse` from the previous
|
||||||
|
// checkout, and a failed checkout aborts before the build.
|
||||||
|
func checkoutScript(buildDir, branch string) string {
|
||||||
|
return "set -eu\n" +
|
||||||
|
"git -C " + shellQuote(buildDir) + " diff --quiet || { echo 'working tree has uncommitted changes; refusing to overwrite' >&2; exit 1; }\n" +
|
||||||
|
"git -C " + shellQuote(buildDir) + " diff --cached --quiet || { echo 'working tree has staged changes; refusing to overwrite' >&2; exit 1; }\n" +
|
||||||
|
"git -C " + shellQuote(buildDir) + " checkout -q -B " + shellQuote(branch) + " " + shellQuote("origin/"+branch) + "\n" +
|
||||||
|
"git -C " + shellQuote(buildDir) + " rev-parse HEAD\n"
|
||||||
|
}
|
||||||
|
|
||||||
|
// commitRecordScript writes the resolved branch and commit to the build
|
||||||
|
// dir record file with a bounded, greppable format. It runs only after a
|
||||||
|
// successful build, so the record always matches the staged binary.
|
||||||
|
func commitRecordScript(buildDir, branch, commit string) string {
|
||||||
|
path := shellQuote(filepath.Join(buildDir, commitRecordName))
|
||||||
|
format := shellQuote("branch=%s\\ncommit=%s\\n")
|
||||||
|
return "umask 022; printf " + format + " " + shellQuote(branch) + " " + shellQuote(commit) +
|
||||||
|
" > " + path + " && chmod 0644 " + path
|
||||||
|
}
|
||||||
|
|
||||||
|
// buildScript builds the worker with the repository's own flags into the
|
||||||
|
// staging path and verifies the resulting binary runs. CGO is disabled,
|
||||||
|
// trimpath keeps the build reproducible, and both caches (GOCACHE and
|
||||||
|
// GOMODCACHE) live inside the build dir so reruns reuse them. The binary
|
||||||
|
// is built to a sibling temp path, verified, and only then atomically
|
||||||
|
// swapped over the previous staging binary, so a failed build never
|
||||||
|
// replaces it.
|
||||||
|
func buildScript(toolchainDir, buildDir, goproxy, stage, ldflags string) string {
|
||||||
|
var b strings.Builder
|
||||||
|
b.WriteString("set -eu\n")
|
||||||
|
b.WriteString("cd " + shellQuote(buildDir) + "\n")
|
||||||
|
b.WriteString("export PATH=" + shellQuote(toolchainDir+"/bin") + ":$PATH\n")
|
||||||
|
b.WriteString("export GOCACHE=" + shellQuote(buildDir+"/.gocache") + "\n")
|
||||||
|
b.WriteString("export GOMODCACHE=" + shellQuote(buildDir+"/.gomodcache") + "\n")
|
||||||
|
if goproxy != "" {
|
||||||
|
b.WriteString("export GOPROXY=" + shellQuote(goproxy) + "\n")
|
||||||
|
}
|
||||||
|
b.WriteString("stage=" + shellQuote(stage) + "\n")
|
||||||
|
b.WriteString("tmp=\"$stage.new\"\n")
|
||||||
|
b.WriteString("trap 'rm -f \"$tmp\"' EXIT HUP INT TERM\n")
|
||||||
|
b.WriteString("CGO_ENABLED=0 ")
|
||||||
|
b.WriteString(shellQuote(toolchainDir + "/bin/go"))
|
||||||
|
b.WriteString(" build -trimpath -ldflags=" + shellQuote(ldflags) + " -o \"$tmp\" ./cmd/rsmon-worker\n")
|
||||||
|
b.WriteString("\"$tmp\" --version\n")
|
||||||
|
b.WriteString("mv -f \"$tmp\" \"$stage\"\n")
|
||||||
|
return b.String()
|
||||||
|
}
|
||||||
536
internal/installer/sourceinstall_ssh_test.go
Обычный файл
536
internal/installer/sourceinstall_ssh_test.go
Обычный файл
@@ -0,0 +1,536 @@
|
|||||||
|
package installer
|
||||||
|
|
||||||
|
import (
|
||||||
|
"crypto/ed25519"
|
||||||
|
"crypto/rand"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"net"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"golang.org/x/crypto/ssh"
|
||||||
|
|
||||||
|
"rocketgit.ru/rsmon/worker/internal/sshinstall"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
fakeSSHPassword = "fake-ssh-password"
|
||||||
|
fakeCommitHex = "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef"
|
||||||
|
fakeOsRelease = "ID=ubuntu\nNAME=\"Ubuntu\"\nVERSION_ID=24.04\n"
|
||||||
|
)
|
||||||
|
|
||||||
|
// fakeSSHServer is a minimal in-process SSH server that simulates a
|
||||||
|
// remote Linux host for the SourceInstall orchestration tests. It uses
|
||||||
|
// real golang.org/x/crypto/ssh transport (no mocked SSH library), so the
|
||||||
|
// executor's dial, session, exec, and stdout/stderr plumbing is
|
||||||
|
// exercised end to end, and it records every command it ran.
|
||||||
|
type fakeSSHServer struct {
|
||||||
|
addr string
|
||||||
|
onExec func(command string) (stdout, stderr string, code int)
|
||||||
|
mu sync.Mutex
|
||||||
|
commands []string
|
||||||
|
}
|
||||||
|
|
||||||
|
func startFakeSSHServer(t *testing.T, onExec func(command string) (string, string, int)) *fakeSSHServer {
|
||||||
|
t.Helper()
|
||||||
|
_, priv, err := ed25519.GenerateKey(rand.Reader)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
signer, err := ssh.NewSignerFromKey(priv)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
config := &ssh.ServerConfig{
|
||||||
|
PasswordCallback: func(_ ssh.ConnMetadata, pass []byte) (*ssh.Permissions, error) {
|
||||||
|
if string(pass) == fakeSSHPassword {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
return nil, fmt.Errorf("password rejected")
|
||||||
|
},
|
||||||
|
}
|
||||||
|
config.AddHostKey(signer)
|
||||||
|
|
||||||
|
ln, err := net.Listen("tcp", "127.0.0.1:0")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Cleanup(func() { _ = ln.Close() })
|
||||||
|
|
||||||
|
srv := &fakeSSHServer{addr: ln.Addr().String(), onExec: onExec}
|
||||||
|
go func() {
|
||||||
|
for {
|
||||||
|
conn, err := ln.Accept()
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
go srv.handleConn(conn, config)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
return srv
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *fakeSSHServer) Port() int {
|
||||||
|
_, port, err := net.SplitHostPort(s.addr)
|
||||||
|
if err != nil {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
p := 0
|
||||||
|
fmt.Sscanf(port, "%d", &p)
|
||||||
|
return p
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *fakeSSHServer) Commands() []string {
|
||||||
|
s.mu.Lock()
|
||||||
|
defer s.mu.Unlock()
|
||||||
|
return append([]string(nil), s.commands...)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *fakeSSHServer) handleConn(conn net.Conn, config *ssh.ServerConfig) {
|
||||||
|
sconn, chans, reqs, err := ssh.NewServerConn(conn, config)
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
defer sconn.Close() //nolint:errcheck
|
||||||
|
go ssh.DiscardRequests(reqs)
|
||||||
|
for newChannel := range chans {
|
||||||
|
if newChannel.ChannelType() != "session" {
|
||||||
|
_ = newChannel.Reject(ssh.UnknownChannelType, "unknown channel type")
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
channel, requests, err := newChannel.Accept()
|
||||||
|
if err != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
go func() {
|
||||||
|
defer channel.Close()
|
||||||
|
// Drain the client's stdin so uploads (which stream base64
|
||||||
|
// over the session stdin) never block the channel window.
|
||||||
|
// The channel is not closed until the client's write side is
|
||||||
|
// exhausted, mirroring the real server behavior.
|
||||||
|
var drained sync.WaitGroup
|
||||||
|
drained.Add(1)
|
||||||
|
go func() {
|
||||||
|
defer drained.Done()
|
||||||
|
_, _ = io.Copy(io.Discard, channel)
|
||||||
|
}()
|
||||||
|
s.handleSession(channel, requests)
|
||||||
|
drained.Wait()
|
||||||
|
}()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *fakeSSHServer) handleSession(channel ssh.Channel, requests <-chan *ssh.Request) {
|
||||||
|
for req := range requests {
|
||||||
|
if req.Type != "exec" {
|
||||||
|
if req.WantReply {
|
||||||
|
_ = req.Reply(false, nil)
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
var payload struct{ Command string }
|
||||||
|
if err := ssh.Unmarshal(req.Payload, &payload); err != nil {
|
||||||
|
if req.WantReply {
|
||||||
|
_ = req.Reply(false, nil)
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if req.WantReply {
|
||||||
|
_ = req.Reply(true, nil)
|
||||||
|
}
|
||||||
|
s.mu.Lock()
|
||||||
|
s.commands = append(s.commands, payload.Command)
|
||||||
|
s.mu.Unlock()
|
||||||
|
s.execCommand(channel, payload.Command)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *fakeSSHServer) execCommand(channel ssh.Channel, command string) {
|
||||||
|
handler := s.onExec
|
||||||
|
if handler == nil {
|
||||||
|
handler = defaultFakeExec
|
||||||
|
}
|
||||||
|
stdout, stderr, code := handler(command)
|
||||||
|
_, _ = channel.Write([]byte(stdout))
|
||||||
|
_, _ = channel.Stderr().Write([]byte(stderr))
|
||||||
|
_, _ = channel.SendRequest("exit-status", false, ssh.Marshal(struct{ Status uint32 }{uint32(code)}))
|
||||||
|
_ = channel.CloseWrite()
|
||||||
|
}
|
||||||
|
|
||||||
|
// defaultFakeExec simulates a minimal Linux host: it answers os-release,
|
||||||
|
// uname, the init-marker file probe, origin URL, branch resolution, and
|
||||||
|
// rev-parse, accepts every install step, and simulates the work-package-4
|
||||||
|
// activation (running-init probe, env/unit uploads, and the activate
|
||||||
|
// script succeeding). A pinned-branch existence check (show-ref) fails by
|
||||||
|
// default so the missing-branch path is exercised without extra setup.
|
||||||
|
func defaultFakeExec(command string) (string, string, int) {
|
||||||
|
switch {
|
||||||
|
case strings.Contains(command, "cat /etc/os-release"):
|
||||||
|
return fakeOsRelease, "", 0
|
||||||
|
case strings.Contains(command, "uname -m"):
|
||||||
|
return "x86_64\n", "", 0
|
||||||
|
case strings.Contains(command, "[ -e"):
|
||||||
|
return "/usr/lib/systemd/system\n", "", 0
|
||||||
|
case strings.Contains(command, "remote.origin.url"):
|
||||||
|
return sshinstall.DefaultRepo + "\n", "", 0
|
||||||
|
case strings.Contains(command, "symbolic-ref"):
|
||||||
|
return "origin/master\n", "", 0
|
||||||
|
case strings.Contains(command, "rev-parse HEAD"):
|
||||||
|
return fakeCommitHex + "\n", "", 0
|
||||||
|
case strings.Contains(command, "show-ref"):
|
||||||
|
return "", "branch not found", 1
|
||||||
|
case strings.Contains(command, "softlevel"):
|
||||||
|
// running-init probe: no systemd/openrc is actually booted.
|
||||||
|
return "none\n", "", 0
|
||||||
|
case strings.Contains(command, "mktemp -d /tmp/rsmon-worker-act"):
|
||||||
|
// server-side secure 0700 upload dir.
|
||||||
|
return "/tmp/rsmon-worker-act-fake\n", "", 0
|
||||||
|
case strings.Contains(command, "base64 -d"):
|
||||||
|
// env/unit uploads succeed.
|
||||||
|
return "", "", 0
|
||||||
|
case strings.Contains(command, "rsmon-worker activated"):
|
||||||
|
// the activate script completed successfully.
|
||||||
|
return "", "", 0
|
||||||
|
default:
|
||||||
|
return "", "", 0
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func testSSHOptions(port int) SourceInstallOptions {
|
||||||
|
return SourceInstallOptions{
|
||||||
|
SSHOptions: SSHOptions{
|
||||||
|
Host: "127.0.0.1",
|
||||||
|
Port: port,
|
||||||
|
User: "root",
|
||||||
|
Password: fakeSSHPassword,
|
||||||
|
InsecureHostKey: true,
|
||||||
|
},
|
||||||
|
// Staging-only by default so the work-package-3 orchestration
|
||||||
|
// tests keep their historical shape; activation tests opt in.
|
||||||
|
Activation: ActivationOptions{Activate: false},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// testActivationOptions wraps testSSHOptions with the credentials and
|
||||||
|
// activation flag needed to run the work-package-4 flow against the fake
|
||||||
|
// server.
|
||||||
|
func testActivationOptions(port int) SourceInstallOptions {
|
||||||
|
opts := testSSHOptions(port)
|
||||||
|
opts.Activation = ActivationOptions{
|
||||||
|
Activate: true,
|
||||||
|
URL: "https://rsmon.ru",
|
||||||
|
Token: fakeSSHPassword + "-token",
|
||||||
|
}
|
||||||
|
return opts
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSourceInstallSSHFlow(t *testing.T) {
|
||||||
|
srv := startFakeSSHServer(t, nil)
|
||||||
|
res, err := SourceInstall(testSSHOptions(srv.Port()))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if res.Detection.Distro != sshinstall.DistroUbuntu || res.Detection.PackageManager != sshinstall.PkgApt {
|
||||||
|
t.Fatalf("detection = %+v", res.Detection)
|
||||||
|
}
|
||||||
|
if res.Detection.InitSystem != sshinstall.InitSystemd {
|
||||||
|
t.Fatalf("init detection = %q, want systemd", res.Detection.InitSystem)
|
||||||
|
}
|
||||||
|
if res.Plan.Toolchain.Arch != "linux-amd64" || res.Plan.Toolchain.Version != "1.26.0" {
|
||||||
|
t.Fatalf("toolchain = %+v", res.Plan.Toolchain)
|
||||||
|
}
|
||||||
|
if len(res.Plan.Packages) == 0 || res.Plan.Repo == "" {
|
||||||
|
t.Fatalf("plan = %+v", res.Plan)
|
||||||
|
}
|
||||||
|
if res.ResolvedBranch != "master" || res.ResolvedCommit != fakeCommitHex {
|
||||||
|
t.Fatalf("resolved = %s @ %s", res.ResolvedBranch, res.ResolvedCommit)
|
||||||
|
}
|
||||||
|
if res.ToolchainDir != "/usr/local/go" || res.StageBinary != "/opt/rsmon-worker-src/rsmon-worker" ||
|
||||||
|
res.RecordFile != "/opt/rsmon-worker-src/rsmon-worker.commit" {
|
||||||
|
t.Fatalf("paths = %+v", res)
|
||||||
|
}
|
||||||
|
|
||||||
|
commands := srv.Commands()
|
||||||
|
var joined strings.Builder
|
||||||
|
for _, c := range commands {
|
||||||
|
joined.WriteString(c)
|
||||||
|
joined.WriteString("\n")
|
||||||
|
}
|
||||||
|
// Marker substrings that survive the nested `sh -c '<script>'`
|
||||||
|
// quoting; exact quoting of each script is asserted by the unit
|
||||||
|
// tests (TestPackageScript, TestToolchainScript, ...).
|
||||||
|
for _, want := range []string{
|
||||||
|
"cat /etc/os-release",
|
||||||
|
"uname -m",
|
||||||
|
"apt-get update",
|
||||||
|
"apt-get install -y --no-install-recommends",
|
||||||
|
"sha256sum -c -",
|
||||||
|
"git clone",
|
||||||
|
"git -C",
|
||||||
|
"fetch --prune origin",
|
||||||
|
"symbolic-ref --short refs/remotes/origin/HEAD",
|
||||||
|
"checkout -q -B",
|
||||||
|
"rev-parse HEAD",
|
||||||
|
"branch=%s\\ncommit=%s\\n",
|
||||||
|
fakeCommitHex,
|
||||||
|
"CGO_ENABLED=0",
|
||||||
|
"build -trimpath",
|
||||||
|
"GOMODCACHE",
|
||||||
|
"mv -f",
|
||||||
|
} {
|
||||||
|
if !strings.Contains(joined.String(), want) {
|
||||||
|
t.Fatalf("recorded commands missing %q:\n%s", want, joined.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Ordering: prerequisites before toolchain before source before build.
|
||||||
|
idx := func(sub string) int {
|
||||||
|
for i, c := range commands {
|
||||||
|
if strings.Contains(c, sub) {
|
||||||
|
return i
|
||||||
|
}
|
||||||
|
}
|
||||||
|
t.Fatalf("command %q not found in %v", sub, commands)
|
||||||
|
return -1
|
||||||
|
}
|
||||||
|
if !(idx("apt-get install") < idx("sha256sum") && idx("sha256sum") < idx("git clone") &&
|
||||||
|
idx("git clone") < idx("rev-parse") && idx("rev-parse") < idx("-trimpath") &&
|
||||||
|
idx("-trimpath") < idx("branch=%s")) {
|
||||||
|
t.Fatalf("step order wrong: %v", commands)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Adaptive resolution means no pinned-branch existence check ran.
|
||||||
|
if strings.Contains(joined.String(), "show-ref") {
|
||||||
|
t.Fatalf("show-ref ran despite branch resolution:\n%s", joined.String())
|
||||||
|
}
|
||||||
|
if strings.Contains(joined.String(), fakeSSHPassword) {
|
||||||
|
t.Fatal("SSH password leaked into a remote command")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSourceInstallSSHRejectsMissingPinnedBranch(t *testing.T) {
|
||||||
|
srv := startFakeSSHServer(t, nil)
|
||||||
|
opts := testSSHOptions(srv.Port())
|
||||||
|
opts.Branch = "main"
|
||||||
|
_, err := SourceInstall(opts)
|
||||||
|
if err == nil || !strings.Contains(err.Error(), `branch "main" does not exist`) {
|
||||||
|
t.Fatalf("err = %v, want missing-branch error", err)
|
||||||
|
}
|
||||||
|
if commands := srv.Commands(); !strings.Contains(strings.Join(commands, "\n"), "show-ref") {
|
||||||
|
t.Fatalf("pinned branch existence was not verified: %v", commands)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSourceInstallSSHBuildFailure(t *testing.T) {
|
||||||
|
srv := startFakeSSHServer(t, func(command string) (string, string, int) {
|
||||||
|
if strings.Contains(command, "-trimpath") {
|
||||||
|
return "", "build exploded", 1
|
||||||
|
}
|
||||||
|
return defaultFakeExec(command)
|
||||||
|
})
|
||||||
|
_, err := SourceInstall(testSSHOptions(srv.Port()))
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "build worker binary") || !strings.Contains(err.Error(), "build exploded") {
|
||||||
|
t.Fatalf("err = %v, want bounded build failure", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSourceInstallSSHDetectionFailure(t *testing.T) {
|
||||||
|
srv := startFakeSSHServer(t, func(command string) (string, string, int) {
|
||||||
|
if strings.Contains(command, "os-release") {
|
||||||
|
return "", "os-release unreadable", 1
|
||||||
|
}
|
||||||
|
return defaultFakeExec(command)
|
||||||
|
})
|
||||||
|
_, err := SourceInstall(testSSHOptions(srv.Port()))
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "/etc/os-release") {
|
||||||
|
t.Fatalf("err = %v, want detection failure", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestSourceInstallSSHCheckoutFailureNotMasked proves the fail-closed
|
||||||
|
// contract: a failed checkout (e.g. a dirty working tree) surfaces as an
|
||||||
|
// error and never reaches the build or commit-record steps, so the
|
||||||
|
// previous staging binary and record are preserved.
|
||||||
|
func TestSourceInstallSSHCheckoutFailureNotMasked(t *testing.T) {
|
||||||
|
srv := startFakeSSHServer(t, func(command string) (string, string, int) {
|
||||||
|
if strings.Contains(command, "checkout -q -B") {
|
||||||
|
return "", "your local changes to the following files would be overwritten by checkout", 1
|
||||||
|
}
|
||||||
|
return defaultFakeExec(command)
|
||||||
|
})
|
||||||
|
_, err := SourceInstall(testSSHOptions(srv.Port()))
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "check out branch") {
|
||||||
|
t.Fatalf("err = %v, want checkout failure", err)
|
||||||
|
}
|
||||||
|
joined := strings.Join(srv.Commands(), "\n")
|
||||||
|
if strings.Contains(joined, "-trimpath") || strings.Contains(joined, "branch=%s") {
|
||||||
|
t.Fatalf("build or record ran after checkout failed:\n%s", joined)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestSourceInstallSSHCommandTimeout verifies each remote command is
|
||||||
|
// bounded by SessionTimeout and the run reports it.
|
||||||
|
func TestSourceInstallSSHCommandTimeout(t *testing.T) {
|
||||||
|
srv := startFakeSSHServer(t, func(command string) (string, string, int) {
|
||||||
|
if strings.Contains(command, "uname -m") {
|
||||||
|
time.Sleep(5 * time.Second)
|
||||||
|
return "x86_64\n", "", 0
|
||||||
|
}
|
||||||
|
return defaultFakeExec(command)
|
||||||
|
})
|
||||||
|
opts := testSSHOptions(srv.Port())
|
||||||
|
opts.SessionTimeout = 300 * time.Millisecond
|
||||||
|
_, err := SourceInstall(opts)
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "timed out after") {
|
||||||
|
t.Fatalf("err = %v, want command timeout", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestSourceInstallSSHActivationFlow verifies the work-package-4 flow
|
||||||
|
// over a real SSH session: after the staging build and commit record, the
|
||||||
|
// installer probes the running init, uploads the env and service
|
||||||
|
// definition, runs the atomic activation script, and reports the installed
|
||||||
|
// layout. The secrets-absent contract holds: the worker token never
|
||||||
|
// reaches a remote command.
|
||||||
|
func TestSourceInstallSSHActivationFlow(t *testing.T) {
|
||||||
|
srv := startFakeSSHServer(t, nil)
|
||||||
|
res, err := SourceInstall(testActivationOptions(srv.Port()))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if res.Activation == nil {
|
||||||
|
t.Fatal("activation result missing")
|
||||||
|
}
|
||||||
|
if res.Activation.Binary != "/usr/local/bin/rsmon-worker" ||
|
||||||
|
res.Activation.EnvFile != "/etc/rsmon-worker/worker.env" ||
|
||||||
|
res.Activation.DataDir != "/var/lib/rsmon-worker" ||
|
||||||
|
res.Activation.UnitFile != "/etc/systemd/system/rsmon-worker.service" ||
|
||||||
|
res.Activation.Supervisor != "none" ||
|
||||||
|
!res.Activation.Started {
|
||||||
|
t.Fatalf("activation result = %+v", res.Activation)
|
||||||
|
}
|
||||||
|
|
||||||
|
commands := srv.Commands()
|
||||||
|
var joined strings.Builder
|
||||||
|
for _, c := range commands {
|
||||||
|
joined.WriteString(c)
|
||||||
|
joined.WriteString("\n")
|
||||||
|
}
|
||||||
|
for _, want := range []string{
|
||||||
|
"/run/systemd/system",
|
||||||
|
"softlevel",
|
||||||
|
"mktemp -d /tmp/rsmon-worker-act",
|
||||||
|
"umask 077; base64 -d >",
|
||||||
|
"rsmon-worker activated",
|
||||||
|
"install -m 0755",
|
||||||
|
"install -m 0600",
|
||||||
|
"chmod 0750",
|
||||||
|
"worker.pid",
|
||||||
|
"liveness",
|
||||||
|
"rm -rf --",
|
||||||
|
} {
|
||||||
|
if !strings.Contains(joined.String(), want) {
|
||||||
|
t.Fatalf("activation commands missing %q:\n%s", want, joined.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Ordering: build and commit record, then a server-side secure upload
|
||||||
|
// dir is created, the env/unit are uploaded into it, and the activation
|
||||||
|
// script runs last.
|
||||||
|
idx := func(sub string) int {
|
||||||
|
for i, c := range commands {
|
||||||
|
if strings.Contains(c, sub) {
|
||||||
|
return i
|
||||||
|
}
|
||||||
|
}
|
||||||
|
t.Fatalf("command %q not found in %v", sub, commands)
|
||||||
|
return -1
|
||||||
|
}
|
||||||
|
if idx("branch=%s") >= idx("mktemp -d /tmp/rsmon-worker-act") ||
|
||||||
|
idx("mktemp -d /tmp/rsmon-worker-act") >= idx("base64 -d >") ||
|
||||||
|
idx("base64 -d >") >= idx("rsmon-worker activated") {
|
||||||
|
t.Fatalf("activation order wrong: %v", commands)
|
||||||
|
}
|
||||||
|
if strings.Contains(joined.String(), fakeSSHPassword+"-token") {
|
||||||
|
t.Fatal("worker token leaked into a remote command")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestSourceInstallSSHNoActivationSkipsService proves the staging
|
||||||
|
// boundary: without Activation the installer never probes the init system,
|
||||||
|
// uploads an env/unit, or runs the activation script.
|
||||||
|
func TestSourceInstallSSHNoActivationSkipsService(t *testing.T) {
|
||||||
|
srv := startFakeSSHServer(t, nil)
|
||||||
|
res, err := SourceInstall(testSSHOptions(srv.Port()))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if res.Activation != nil {
|
||||||
|
t.Fatalf("activation must not run without the flag: %+v", res.Activation)
|
||||||
|
}
|
||||||
|
joined := strings.Join(srv.Commands(), "\n")
|
||||||
|
for _, forbidden := range []string{"softlevel", "base64 -d >", "rsmon-worker activated", "worker.pid"} {
|
||||||
|
if strings.Contains(joined, forbidden) {
|
||||||
|
t.Fatalf("staging-only flow ran activation step %q:\n%s", forbidden, joined)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestSourceInstallSSHActivationRequiresToken verifies the credentials
|
||||||
|
// gate: activation without a token fails before any remote connection.
|
||||||
|
func TestSourceInstallSSHActivationRequiresToken(t *testing.T) {
|
||||||
|
srv := startFakeSSHServer(t, nil)
|
||||||
|
opts := testSSHOptions(srv.Port())
|
||||||
|
opts.Activation = ActivationOptions{Activate: true, URL: "https://rsmon.ru"}
|
||||||
|
_, err := SourceInstall(opts)
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "RSMON_TOKEN") {
|
||||||
|
t.Fatalf("err = %v, want token requirement", err)
|
||||||
|
}
|
||||||
|
if got := len(srv.Commands()); got != 0 {
|
||||||
|
t.Fatalf("commands ran before validation failed: %d", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestSourceInstallSSHActivationFailure verifies the bounded failure
|
||||||
|
// path: a failing activate script surfaces as a labelled error with the
|
||||||
|
// remote stderr, and the uploaded temp env/unit are cleaned up by the
|
||||||
|
// controller defer regardless of the outcome.
|
||||||
|
func TestSourceInstallSSHActivationFailure(t *testing.T) {
|
||||||
|
srv := startFakeSSHServer(t, func(command string) (string, string, int) {
|
||||||
|
if strings.Contains(command, "rsmon-worker activated") {
|
||||||
|
return "", "activation exploded: disk full", 1
|
||||||
|
}
|
||||||
|
return defaultFakeExec(command)
|
||||||
|
})
|
||||||
|
_, err := SourceInstall(testActivationOptions(srv.Port()))
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "activate worker service") || !strings.Contains(err.Error(), "disk full") {
|
||||||
|
t.Fatalf("err = %v, want bounded activation failure", err)
|
||||||
|
}
|
||||||
|
joined := strings.Join(srv.Commands(), "\n")
|
||||||
|
if !strings.Contains(joined, "rm -rf --") {
|
||||||
|
t.Fatalf("secure upload dir cleanup not issued after activation failure:\n%s", joined)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestSourceInstallSSHActivationHealthFailure verifies the health gate
|
||||||
|
// surfaces as an activation failure and the temp files are still cleaned.
|
||||||
|
// The fake cannot run the shell script, so the failure is injected at the
|
||||||
|
// activate-command level; the real rollback semantics are covered by the
|
||||||
|
// Docker/OpenSSH E2E tests.
|
||||||
|
func TestSourceInstallSSHActivationHealthFailure(t *testing.T) {
|
||||||
|
srv := startFakeSSHServer(t, func(command string) (string, string, int) {
|
||||||
|
if strings.Contains(command, "rsmon-worker activated") {
|
||||||
|
return "", "health failure: worker did not answer /healthz", 1
|
||||||
|
}
|
||||||
|
return defaultFakeExec(command)
|
||||||
|
})
|
||||||
|
_, err := SourceInstall(testActivationOptions(srv.Port()))
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "did not answer /healthz") {
|
||||||
|
t.Fatalf("err = %v, want /healthz verification failure", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
395
internal/installer/sourceinstall_test.go
Обычный файл
395
internal/installer/sourceinstall_test.go
Обычный файл
@@ -0,0 +1,395 @@
|
|||||||
|
package installer
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"rocketgit.ru/rsmon/worker/internal/sshinstall"
|
||||||
|
)
|
||||||
|
|
||||||
|
// cannedSHA is a fixed 64-hex value used to exercise script rendering.
|
||||||
|
const cannedSHA = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
|
||||||
|
|
||||||
|
func TestPackageScript(t *testing.T) {
|
||||||
|
pkgs := []string{"git", "ca-certificates", "curl", "tar", "gzip"}
|
||||||
|
|
||||||
|
apk := packageScript(sshinstall.PkgApk, pkgs)
|
||||||
|
if !strings.HasPrefix(apk, "apk add --no-cache ") {
|
||||||
|
t.Fatalf("apk script = %q", apk)
|
||||||
|
}
|
||||||
|
for _, p := range pkgs {
|
||||||
|
if !strings.Contains(apk, shellQuote(p)) {
|
||||||
|
t.Fatalf("apk script missing quoted package %q: %q", p, apk)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
apt := packageScript(sshinstall.PkgApt, pkgs)
|
||||||
|
for _, want := range []string{"export DEBIAN_FRONTEND=noninteractive", "apt-get update", "apt-get install -y --no-install-recommends"} {
|
||||||
|
if !strings.Contains(apt, want) {
|
||||||
|
t.Fatalf("apt script missing %q: %q", want, apt)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pacman := packageScript(sshinstall.PkgPacman, pkgs)
|
||||||
|
if !strings.Contains(pacman, "pacman -Sy --noconfirm --needed") {
|
||||||
|
t.Fatalf("pacman script = %q", pacman)
|
||||||
|
}
|
||||||
|
|
||||||
|
dnf := packageScript(sshinstall.PkgDnf, pkgs)
|
||||||
|
if !strings.Contains(dnf, "dnf install -y") {
|
||||||
|
t.Fatalf("dnf script = %q", dnf)
|
||||||
|
}
|
||||||
|
|
||||||
|
if got := packageScript(sshinstall.PkgUnknown, pkgs); got != "" {
|
||||||
|
t.Fatalf("unknown pkg script = %q, want empty", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPackageScriptNeverIncludesCompiler(t *testing.T) {
|
||||||
|
pkgs := []string{"git", "ca-certificates", "curl", "tar", "gzip"}
|
||||||
|
for _, pkg := range []sshinstall.PackageManager{sshinstall.PkgApk, sshinstall.PkgApt, sshinstall.PkgPacman, sshinstall.PkgDnf} {
|
||||||
|
script := packageScript(pkg, pkgs)
|
||||||
|
for _, bad := range []string{"build-essential", "gcc", "g++", "base-devel", "make", "gcc-c++"} {
|
||||||
|
if strings.Contains(script, bad) {
|
||||||
|
t.Fatalf("script for %s includes compiler hint %q: %q", pkg, bad, script)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestToolchainScript(t *testing.T) {
|
||||||
|
tc := sshinstall.Toolchain{
|
||||||
|
Version: "1.26.0",
|
||||||
|
Arch: "linux-amd64",
|
||||||
|
URL: "https://go.dev/dl/go1.26.0.linux-amd64.tar.gz",
|
||||||
|
SHA256: cannedSHA,
|
||||||
|
}
|
||||||
|
script := toolchainScript(tc, "/usr/local/go")
|
||||||
|
for _, want := range []string{
|
||||||
|
"set -eu",
|
||||||
|
"mktemp -d /tmp/rsmon-toolchain-XXXXXX",
|
||||||
|
"mktemp -d \"$parent/.go-staging-XXXXXX\"",
|
||||||
|
"trap 'rm -rf \"$work\" \"$staging\"' EXIT HUP INT TERM",
|
||||||
|
"curl -fsSL --retry 3 --retry-delay 2 -o \"$archive\" 'https://go.dev/dl/go1.26.0.linux-amd64.tar.gz'",
|
||||||
|
"sha256sum -c -",
|
||||||
|
cannedSHA,
|
||||||
|
"tar -C \"$staging\" -xzf \"$archive\"",
|
||||||
|
"staged=\"$($staging/go/bin/go version | awk '{print $3}')\"",
|
||||||
|
"backup=\"$parent/.go-backup\"",
|
||||||
|
"mv \"$toolchain\" \"$backup\"",
|
||||||
|
"mv \"$staging/go\" \"$toolchain\"",
|
||||||
|
"\"$toolchain/bin/go\" version",
|
||||||
|
"go toolchain already present",
|
||||||
|
} {
|
||||||
|
if !strings.Contains(script, want) {
|
||||||
|
t.Fatalf("toolchain script missing %q:\n%s", want, script)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !strings.Contains(script, shellQuote("go1.26.0")) {
|
||||||
|
t.Fatalf("toolchain script missing version guard:\n%s", script)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestToolchainScriptIdempotentSkipOnlyForMatchingVersion(t *testing.T) {
|
||||||
|
script := toolchainScript(sshinstall.Toolchain{Version: "1.26.0", Arch: "linux-amd64", URL: "u", SHA256: cannedSHA}, "/usr/local/go")
|
||||||
|
if strings.Count(script, "exit 0") != 1 {
|
||||||
|
t.Fatalf("toolchain script should skip only once:\n%s", script)
|
||||||
|
}
|
||||||
|
// Replacement must be atomic: the prior toolchain is moved aside and
|
||||||
|
// restored when the swap fails, never removed before staging is ready.
|
||||||
|
for _, want := range []string{
|
||||||
|
"if [ -e \"$toolchain\" ]; then",
|
||||||
|
"mv \"$toolchain\" \"$backup\"",
|
||||||
|
"if ! mv \"$staging/go\" \"$toolchain\"; then",
|
||||||
|
"mv \"$backup\" \"$toolchain\"",
|
||||||
|
} {
|
||||||
|
if !strings.Contains(script, want) {
|
||||||
|
t.Fatalf("toolchain script missing %q:\n%s", want, script)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if strings.Contains(script, "rm -rf \"$toolchain\"") {
|
||||||
|
t.Fatalf("toolchain script must not delete the live toolchain directly:\n%s", script)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCloneUpdateScript(t *testing.T) {
|
||||||
|
script := cloneUpdateScript("https://example.test/worker.git", "/opt/rsmon-worker-src")
|
||||||
|
for _, want := range []string{
|
||||||
|
"set -u",
|
||||||
|
"[ ! -d \"$dir/.git\" ]",
|
||||||
|
"git clone \"$repo\" \"$dir\"",
|
||||||
|
"while [ \"$attempt\" -lt 3 ]",
|
||||||
|
"sleep 2",
|
||||||
|
"git -C \"$dir\" config --get remote.origin.url",
|
||||||
|
"existing checkout origin does not match configured repository",
|
||||||
|
"git -C \"$dir\" fetch --prune origin",
|
||||||
|
"exit 1",
|
||||||
|
} {
|
||||||
|
if !strings.Contains(script, want) {
|
||||||
|
t.Fatalf("clone/update script missing %q:\n%s", want, script)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCheckoutScriptFailsClosed(t *testing.T) {
|
||||||
|
script := checkoutScript("/opt/rsmon-worker-src", "master")
|
||||||
|
if !strings.HasPrefix(script, "set -eu\n") {
|
||||||
|
t.Fatalf("checkout script must fail closed with set -eu:\n%s", script)
|
||||||
|
}
|
||||||
|
for _, want := range []string{
|
||||||
|
"git -C '/opt/rsmon-worker-src' diff --quiet ||",
|
||||||
|
"git -C '/opt/rsmon-worker-src' diff --cached --quiet ||",
|
||||||
|
"refusing to overwrite",
|
||||||
|
"checkout -q -B 'master' 'origin/master'",
|
||||||
|
"rev-parse HEAD",
|
||||||
|
} {
|
||||||
|
if !strings.Contains(script, want) {
|
||||||
|
t.Fatalf("checkout script missing %q:\n%s", want, script)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestResolveBranchScriptFailsClosed(t *testing.T) {
|
||||||
|
if !strings.HasPrefix(resolveBranchScript("/opt/rsmon-worker-src"), "set -eu\n") {
|
||||||
|
t.Fatalf("resolve-branch script must fail closed:\n%s", resolveBranchScript("/opt/rsmon-worker-src"))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestResolveBranchScript(t *testing.T) {
|
||||||
|
script := resolveBranchScript("/opt/rsmon-worker-src")
|
||||||
|
for _, want := range []string{
|
||||||
|
"remote set-head origin --auto",
|
||||||
|
"symbolic-ref --short refs/remotes/origin/HEAD",
|
||||||
|
} {
|
||||||
|
if !strings.Contains(script, want) {
|
||||||
|
t.Fatalf("resolve-branch script missing %q:\n%s", want, script)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCheckoutScript(t *testing.T) {
|
||||||
|
script := checkoutScript("/opt/rsmon-worker-src", "master")
|
||||||
|
for _, want := range []string{
|
||||||
|
"checkout -q -B 'master' 'origin/master'",
|
||||||
|
"rev-parse HEAD",
|
||||||
|
} {
|
||||||
|
if !strings.Contains(script, want) {
|
||||||
|
t.Fatalf("checkout script missing %q:\n%s", want, script)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRefExistsScript(t *testing.T) {
|
||||||
|
script := refExistsScript("/opt/rsmon-worker-src", "refs/remotes/origin/main")
|
||||||
|
if !strings.Contains(script, "show-ref --verify --quiet 'refs/remotes/origin/main'") {
|
||||||
|
t.Fatalf("ref-exists script = %q", script)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCommitRecordScript(t *testing.T) {
|
||||||
|
script := commitRecordScript("/opt/rsmon-worker-src", "master", strings.Repeat("a", 40))
|
||||||
|
for _, want := range []string{
|
||||||
|
"branch=%s\\ncommit=%s\\n",
|
||||||
|
"'master'",
|
||||||
|
strings.Repeat("a", 40),
|
||||||
|
"'/opt/rsmon-worker-src/rsmon-worker.commit'",
|
||||||
|
"chmod 0644",
|
||||||
|
} {
|
||||||
|
if !strings.Contains(script, want) {
|
||||||
|
t.Fatalf("record script missing %q:\n%s", want, script)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBuildScript(t *testing.T) {
|
||||||
|
script := buildScript("/usr/local/go", "/opt/rsmon-worker-src", "", "/opt/rsmon-worker-src/rsmon-worker",
|
||||||
|
`-s -w -X main.version=dev -X main.commit=abcdef012345 -X main.buildDate=2026-08-12T00:00:00Z`)
|
||||||
|
for _, want := range []string{
|
||||||
|
"set -eu",
|
||||||
|
"cd '/opt/rsmon-worker-src'",
|
||||||
|
"export PATH='/usr/local/go/bin':$PATH",
|
||||||
|
"export GOCACHE='/opt/rsmon-worker-src/.gocache'",
|
||||||
|
"export GOMODCACHE='/opt/rsmon-worker-src/.gomodcache'",
|
||||||
|
"CGO_ENABLED=0 '/usr/local/go/bin/go' build -trimpath",
|
||||||
|
"-X main.commit=abcdef012345",
|
||||||
|
"-o \"$tmp\" ./cmd/rsmon-worker",
|
||||||
|
"tmp=\"$stage.new\"",
|
||||||
|
"\"$tmp\" --version",
|
||||||
|
"mv -f \"$tmp\" \"$stage\"",
|
||||||
|
"trap 'rm -f \"$tmp\"' EXIT HUP INT TERM",
|
||||||
|
} {
|
||||||
|
if !strings.Contains(script, want) {
|
||||||
|
t.Fatalf("build script missing %q:\n%s", want, script)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if strings.Contains(script, "GOPROXY") {
|
||||||
|
t.Fatalf("empty GOPROXY must not be exported:\n%s", script)
|
||||||
|
}
|
||||||
|
|
||||||
|
withProxy := buildScript("/usr/local/go", "/opt/rsmon-worker-src", "https://proxy.golang.org,direct", "/opt/rsmon-worker-src/rsmon-worker", "-s -w")
|
||||||
|
if !strings.Contains(withProxy, "export GOPROXY='https://proxy.golang.org,direct'") {
|
||||||
|
t.Fatalf("GOPROXY override not rendered:\n%s", withProxy)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBuildScriptVerifiesBeforeSwap(t *testing.T) {
|
||||||
|
script := buildScript("/usr/local/go", "/opt/rsmon-worker-src", "", "/opt/rsmon-worker-src/rsmon-worker", "-s -w")
|
||||||
|
verify := strings.Index(script, "\"$tmp\" --version")
|
||||||
|
swap := strings.Index(script, "mv -f \"$tmp\" \"$stage\"")
|
||||||
|
if verify < 0 || swap < 0 || verify > swap {
|
||||||
|
t.Fatalf("build script must verify the temp binary before swapping it in:\n%s", script)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseResolvedCommit(t *testing.T) {
|
||||||
|
commit := strings.Repeat("abcdef", 6) + "abcd" // 40 hex
|
||||||
|
if got, err := parseResolvedCommit(commit + "\n"); err != nil || got != commit {
|
||||||
|
t.Fatalf("parseResolvedCommit() = %q, %v", got, err)
|
||||||
|
}
|
||||||
|
if got, err := parseResolvedCommit("ignored\n" + commit + "\n"); err != nil || got != commit {
|
||||||
|
t.Fatalf("parseResolvedCommit() multi-line = %q, %v", got, err)
|
||||||
|
}
|
||||||
|
for _, bad := range []string{"", "abc", strings.Repeat("A", 40), strings.Repeat("a", 39), "x" + strings.Repeat("a", 39)} {
|
||||||
|
if _, err := parseResolvedCommit(bad); err == nil {
|
||||||
|
t.Fatalf("parseResolvedCommit(%q) succeeded", bad)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseResolvedBranch(t *testing.T) {
|
||||||
|
if got, err := parseResolvedBranch("origin/master\n"); err != nil || got != "master" {
|
||||||
|
t.Fatalf("parseResolvedBranch(origin/master) = %q, %v", got, err)
|
||||||
|
}
|
||||||
|
if got, err := parseResolvedBranch("master\n"); err != nil || got != "master" {
|
||||||
|
t.Fatalf("parseResolvedBranch(master) = %q, %v", got, err)
|
||||||
|
}
|
||||||
|
// A multi-component short name is legal git and stays safe because
|
||||||
|
// the value is validated and single-quoted everywhere it is used.
|
||||||
|
if got, err := parseResolvedBranch("origin/release/1.0\n"); err != nil || got != "release/1.0" {
|
||||||
|
t.Fatalf("parseResolvedBranch(nested) = %q, %v", got, err)
|
||||||
|
}
|
||||||
|
for _, bad := range []string{"origin/../evil\n", "origin/x y\n", "origin/x..y\n", "origin/x@{y\n", "origin/-x\n"} {
|
||||||
|
if _, err := parseResolvedBranch(bad); err == nil {
|
||||||
|
t.Fatalf("parseResolvedBranch(%q) succeeded", bad)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestValidBranchName(t *testing.T) {
|
||||||
|
for _, ok := range []string{"main", "master", "release-1.0", "feature/x", "a", "v1.2.3", "a_b", "release/1.0"} {
|
||||||
|
if !validBranchName(ok) {
|
||||||
|
t.Fatalf("validBranchName(%q) rejected", ok)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, bad := range []string{"", "-bad", "x..y", "x@{y", "x y", "/x", "x.", "x/", "x//y", "x\\y", "x;y", "$x", "`x`"} {
|
||||||
|
if validBranchName(bad) {
|
||||||
|
t.Fatalf("validBranchName(%q) accepted", bad)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestQuoteList(t *testing.T) {
|
||||||
|
if got, want := quoteList([]string{"git", "ca-certificates"}), "'git' 'ca-certificates'"; got != want {
|
||||||
|
t.Fatalf("quoteList() = %q, want %q", got, want)
|
||||||
|
}
|
||||||
|
if got := quoteList(nil); got != "" {
|
||||||
|
t.Fatalf("quoteList(nil) = %q, want empty", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNormalizeSourceOptions(t *testing.T) {
|
||||||
|
o, err := normalizeSourceOptions(SourceInstallOptions{SSHOptions: SSHOptions{Host: "h", User: "u"}})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if o.Port != 22 || o.ToolchainDir != "/usr/local/go" || o.SessionTimeout != defaultSessionTimeout {
|
||||||
|
t.Fatalf("defaults not applied: %+v", o)
|
||||||
|
}
|
||||||
|
if o, err := normalizeSourceOptions(SourceInstallOptions{SSHOptions: SSHOptions{Host: "h", User: "u"}, SessionTimeout: 7 * time.Minute}); err != nil || o.SessionTimeout != 7*time.Minute {
|
||||||
|
t.Fatalf("explicit session timeout not honored: %+v, %v", o, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tc := range []struct {
|
||||||
|
name string
|
||||||
|
opts SourceInstallOptions
|
||||||
|
}{
|
||||||
|
{name: "missing host", opts: SourceInstallOptions{SSHOptions: SSHOptions{User: "u"}}},
|
||||||
|
{name: "missing user", opts: SourceInstallOptions{SSHOptions: SSHOptions{Host: "h"}}},
|
||||||
|
{name: "bad port", opts: SourceInstallOptions{SSHOptions: SSHOptions{Host: "h", User: "u", Port: 70000}}},
|
||||||
|
{name: "bad branch", opts: SourceInstallOptions{SSHOptions: SSHOptions{Host: "h", User: "u"}, Branch: "x y"}},
|
||||||
|
{name: "bad go version", opts: SourceInstallOptions{SSHOptions: SSHOptions{Host: "h", User: "u"}, GoVersion: "1.26;rm"}},
|
||||||
|
{name: "bad go arch", opts: SourceInstallOptions{SSHOptions: SSHOptions{Host: "h", User: "u"}, GoArch: "amd64;rm"}},
|
||||||
|
{name: "bad repo scheme", opts: SourceInstallOptions{SSHOptions: SSHOptions{Host: "h", User: "u"}, Repo: "http://x/y"}},
|
||||||
|
{name: "repo userinfo", opts: SourceInstallOptions{SSHOptions: SSHOptions{Host: "h", User: "u"}, Repo: "https://user:pass@x/y"}},
|
||||||
|
{name: "relative toolchain", opts: SourceInstallOptions{SSHOptions: SSHOptions{Host: "h", User: "u"}, ToolchainDir: "usr/local/go"}},
|
||||||
|
{name: "non-go toolchain basename", opts: SourceInstallOptions{SSHOptions: SSHOptions{Host: "h", User: "u"}, ToolchainDir: "/opt/golang"}},
|
||||||
|
{name: "relative stage", opts: SourceInstallOptions{SSHOptions: SSHOptions{Host: "h", User: "u"}, StageBinary: "bin/worker"}},
|
||||||
|
} {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
if _, err := normalizeSourceOptions(tc.opts); err == nil {
|
||||||
|
t.Fatalf("normalizeSourceOptions(%s) succeeded", tc.name)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSudoWrap(t *testing.T) {
|
||||||
|
cmd, stdin := sudoWrap("root", "", "echo hi")
|
||||||
|
if cmd != "echo hi" || stdin != nil {
|
||||||
|
t.Fatalf("root wrap = %q, %q", cmd, stdin)
|
||||||
|
}
|
||||||
|
|
||||||
|
cmd, stdin = sudoWrap("deploy", "", "echo hi")
|
||||||
|
if cmd != "sudo -n -- echo hi" || stdin != nil {
|
||||||
|
t.Fatalf("passwordless sudo wrap = %q, %q", cmd, stdin)
|
||||||
|
}
|
||||||
|
|
||||||
|
cmd, stdin = sudoWrap("deploy", "supersecret", "echo hi")
|
||||||
|
if cmd != "sudo -S -p '' -- echo hi" || string(stdin) != "supersecret\n" {
|
||||||
|
t.Fatalf("sudo -S wrap = %q, %q", cmd, stdin)
|
||||||
|
}
|
||||||
|
if strings.Contains(cmd, "supersecret") {
|
||||||
|
t.Fatal("sudo password leaked into the command string")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestSourceScriptsNeverContainSecrets documents the "secrets absent"
|
||||||
|
// contract: none of the rendered remote scripts carry a credential.
|
||||||
|
func TestBoundedBuffer(t *testing.T) {
|
||||||
|
var b boundedBuffer
|
||||||
|
b.max = 8
|
||||||
|
if _, err := b.Write([]byte("12345")); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if b.truncated {
|
||||||
|
t.Fatal("truncated before exceeding max")
|
||||||
|
}
|
||||||
|
if _, err := b.Write([]byte("6789abcdef")); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !b.truncated {
|
||||||
|
t.Fatal("overflow not flagged")
|
||||||
|
}
|
||||||
|
if got, want := b.String(), "12345678"; got != want {
|
||||||
|
t.Fatalf("boundedBuffer = %q, want %q", got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSourceScriptsNeverContainSecrets(t *testing.T) {
|
||||||
|
const secret = "super-secret-token-value"
|
||||||
|
scripts := []string{
|
||||||
|
packageScript(sshinstall.PkgApk, []string{"git", "ca-certificates", "curl", "tar", "gzip"}),
|
||||||
|
toolchainScript(sshinstall.Toolchain{Version: "1.26.0", Arch: "linux-amd64", URL: "https://go.dev/dl/go1.26.0.linux-amd64.tar.gz", SHA256: cannedSHA}, "/usr/local/go"),
|
||||||
|
cloneUpdateScript("https://example.test/worker.git", "/opt/rsmon-worker-src"),
|
||||||
|
resolveBranchScript("/opt/rsmon-worker-src"),
|
||||||
|
checkoutScript("/opt/rsmon-worker-src", "master"),
|
||||||
|
commitRecordScript("/opt/rsmon-worker-src", "master", strings.Repeat("a", 40)),
|
||||||
|
buildScript("/usr/local/go", "/opt/rsmon-worker-src", "", "/opt/rsmon-worker-src/rsmon-worker", "-s -w"),
|
||||||
|
}
|
||||||
|
for i, script := range scripts {
|
||||||
|
if strings.Contains(script, secret) {
|
||||||
|
t.Fatalf("script %d contains a secret", i)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
221
internal/sshinstall/detect.go
Обычный файл
221
internal/sshinstall/detect.go
Обычный файл
@@ -0,0 +1,221 @@
|
|||||||
|
// Package sshinstall contains the pure detection and planning layer for
|
||||||
|
// installing the worker from source over SSH. It never touches the
|
||||||
|
// network or a remote host; the executor that turns a plan into remote
|
||||||
|
// commands is a later work package (docs/source-installation.md).
|
||||||
|
//
|
||||||
|
// The Docker/OpenSSH harness that exercises detection against real
|
||||||
|
// distro containers lives in internal/installer/harness.
|
||||||
|
package sshinstall
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Distro is a supported remote Linux distribution family.
|
||||||
|
type Distro string
|
||||||
|
|
||||||
|
const (
|
||||||
|
DistroUnknown Distro = "unknown"
|
||||||
|
DistroAlpine Distro = "alpine"
|
||||||
|
DistroDebian Distro = "debian"
|
||||||
|
DistroUbuntu Distro = "ubuntu"
|
||||||
|
DistroArch Distro = "arch"
|
||||||
|
DistroCentOS Distro = "centos"
|
||||||
|
DistroFedora Distro = "fedora"
|
||||||
|
DistroRocky Distro = "rocky"
|
||||||
|
DistroAlma Distro = "alma"
|
||||||
|
DistroRHEL Distro = "rhel"
|
||||||
|
)
|
||||||
|
|
||||||
|
// PackageManager is the remote package manager used to install build
|
||||||
|
// prerequisites.
|
||||||
|
type PackageManager string
|
||||||
|
|
||||||
|
const (
|
||||||
|
PkgUnknown PackageManager = ""
|
||||||
|
PkgApk PackageManager = "apk"
|
||||||
|
PkgApt PackageManager = "apt"
|
||||||
|
PkgPacman PackageManager = "pacman"
|
||||||
|
PkgDnf PackageManager = "dnf"
|
||||||
|
PkgYum PackageManager = "yum"
|
||||||
|
)
|
||||||
|
|
||||||
|
// InitSystem is the remote init system. The source installer must know
|
||||||
|
// it before it can install a service definition; Alpine's OpenRC and the
|
||||||
|
// systemd distros take different paths.
|
||||||
|
type InitSystem string
|
||||||
|
|
||||||
|
const (
|
||||||
|
InitUnknown InitSystem = ""
|
||||||
|
InitSystemd InitSystem = "systemd"
|
||||||
|
InitOpenRC InitSystem = "openrc"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Detection is the resolved identity of a remote Linux host.
|
||||||
|
type Detection struct {
|
||||||
|
ID string // os-release ID (lowercase, e.g. "alpine")
|
||||||
|
Name string // os-release NAME (pretty, may be empty)
|
||||||
|
VersionID string // os-release VERSION_ID (may be empty)
|
||||||
|
Distro Distro
|
||||||
|
PackageManager PackageManager
|
||||||
|
InitSystem InitSystem
|
||||||
|
}
|
||||||
|
|
||||||
|
// FileProber reports which of the given absolute paths exist on the
|
||||||
|
// remote host. Detection uses it to distinguish init systems and
|
||||||
|
// packaging hints without parsing the full command surface. A nil
|
||||||
|
// prober behaves as if nothing exists.
|
||||||
|
type FileProber func(paths ...string) map[string]bool
|
||||||
|
|
||||||
|
// Detect resolves the distro, package manager, and init system of a
|
||||||
|
// remote host from its /etc/os-release contents and a path prober.
|
||||||
|
// Pure and deterministic: no command execution happens here.
|
||||||
|
func Detect(osRelease string, probe FileProber) Detection {
|
||||||
|
d := parseOSRelease(osRelease)
|
||||||
|
d.PackageManager = packageManagerFor(d.ID)
|
||||||
|
d.InitSystem = initFor(d.ID, probe)
|
||||||
|
return d
|
||||||
|
}
|
||||||
|
|
||||||
|
// parseOSRelease extracts the fields the installer cares about from the
|
||||||
|
// /etc/os-release text. Unknown keys are ignored.
|
||||||
|
func parseOSRelease(data string) Detection {
|
||||||
|
var d Detection
|
||||||
|
for _, line := range strings.Split(data, "\n") {
|
||||||
|
line = strings.TrimSpace(line)
|
||||||
|
if line == "" || strings.HasPrefix(line, "#") {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
key, value, ok := strings.Cut(line, "=")
|
||||||
|
if !ok {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
// os-release requires KEY=VALUE with no whitespace around the
|
||||||
|
// equals sign. A spaced assignment is a malformed line and must
|
||||||
|
// not be interpreted.
|
||||||
|
if strings.ContainsAny(key, " \t") {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
value = unquoteOSValue(value)
|
||||||
|
switch strings.TrimSpace(key) {
|
||||||
|
case "ID":
|
||||||
|
d.ID = strings.ToLower(strings.TrimSpace(value))
|
||||||
|
case "NAME":
|
||||||
|
d.Name = strings.TrimSpace(value)
|
||||||
|
case "VERSION_ID":
|
||||||
|
d.VersionID = strings.TrimSpace(value)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
d.Distro = distroForID(d.ID)
|
||||||
|
return d
|
||||||
|
}
|
||||||
|
|
||||||
|
// unquoteOSValue strips the surrounding quotes os-release permits and
|
||||||
|
// unescapes the two escapes the spec defines. Unquoted values pass
|
||||||
|
// through unchanged.
|
||||||
|
func unquoteOSValue(value string) string {
|
||||||
|
value = strings.TrimSpace(value)
|
||||||
|
if len(value) >= 2 && (value[0] == '"' || value[0] == '\'') && value[len(value)-1] == value[0] {
|
||||||
|
value = value[1 : len(value)-1]
|
||||||
|
}
|
||||||
|
value = strings.ReplaceAll(value, `\"`, `"`)
|
||||||
|
value = strings.ReplaceAll(value, `\'`, `'`)
|
||||||
|
value = strings.ReplaceAll(value, `\\`, `\`)
|
||||||
|
value = strings.ReplaceAll(value, `\$`, `$`)
|
||||||
|
value = strings.ReplaceAll(value, "`", "")
|
||||||
|
return value
|
||||||
|
}
|
||||||
|
|
||||||
|
// distroForID maps an os-release ID to a supported distro family.
|
||||||
|
func distroForID(id string) Distro {
|
||||||
|
switch id {
|
||||||
|
case "alpine":
|
||||||
|
return DistroAlpine
|
||||||
|
case "debian":
|
||||||
|
return DistroDebian
|
||||||
|
case "ubuntu", "linuxmint", "elementary":
|
||||||
|
return DistroUbuntu
|
||||||
|
case "arch", "archarm", "manjaro", "endeavouros":
|
||||||
|
return DistroArch
|
||||||
|
case "centos":
|
||||||
|
return DistroCentOS
|
||||||
|
case "fedora":
|
||||||
|
return DistroFedora
|
||||||
|
case "rocky":
|
||||||
|
return DistroRocky
|
||||||
|
case "almalinux":
|
||||||
|
return DistroAlma
|
||||||
|
case "rhel", "redhat":
|
||||||
|
return DistroRHEL
|
||||||
|
default:
|
||||||
|
return DistroUnknown
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// packageManagerFor maps an os-release ID to its package manager.
|
||||||
|
// CentOS-family versions differ (dnf on 8+, yum on 7); the planner
|
||||||
|
// defaults to dnf and the CentOS work package will refine it.
|
||||||
|
func packageManagerFor(id string) PackageManager {
|
||||||
|
switch distroForID(id) {
|
||||||
|
case DistroAlpine:
|
||||||
|
return PkgApk
|
||||||
|
case DistroDebian, DistroUbuntu:
|
||||||
|
return PkgApt
|
||||||
|
case DistroArch:
|
||||||
|
return PkgPacman
|
||||||
|
case DistroCentOS, DistroFedora, DistroRocky, DistroAlma, DistroRHEL:
|
||||||
|
return PkgDnf
|
||||||
|
default:
|
||||||
|
return PkgUnknown
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// initFor infers the init system from well-known filesystem markers. In
|
||||||
|
// a container the PID 1 command is not a reliable signal (sshd or the
|
||||||
|
// runtime command runs first), so the installer uses the packaging
|
||||||
|
// markers systemd and OpenRC leave behind.
|
||||||
|
func initFor(_ string, probe FileProber) InitSystem {
|
||||||
|
if probe == nil {
|
||||||
|
return InitUnknown
|
||||||
|
}
|
||||||
|
present := probe(
|
||||||
|
"/run/systemd/system",
|
||||||
|
"/usr/lib/systemd/system",
|
||||||
|
"/etc/systemd/system",
|
||||||
|
"/sbin/openrc",
|
||||||
|
"/etc/init.d",
|
||||||
|
)
|
||||||
|
if present["/run/systemd/system"] || present["/usr/lib/systemd/system"] || present["/etc/systemd/system"] {
|
||||||
|
return InitSystemd
|
||||||
|
}
|
||||||
|
if present["/sbin/openrc"] || present["/etc/init.d"] {
|
||||||
|
return InitOpenRC
|
||||||
|
}
|
||||||
|
return InitUnknown
|
||||||
|
}
|
||||||
|
|
||||||
|
// Summarize returns a single-line, operator-readable description of the
|
||||||
|
// detection result.
|
||||||
|
func (d Detection) Summarize() string {
|
||||||
|
return fmt.Sprintf("%s (id=%s version=%s, pkg=%s, init=%s)",
|
||||||
|
displayName(d), d.ID, displayValue(d.VersionID),
|
||||||
|
displayValue(string(d.PackageManager)), displayValue(string(d.InitSystem)))
|
||||||
|
}
|
||||||
|
|
||||||
|
func displayName(d Detection) string {
|
||||||
|
if d.Name != "" {
|
||||||
|
return d.Name
|
||||||
|
}
|
||||||
|
if d.ID != "" {
|
||||||
|
return d.ID
|
||||||
|
}
|
||||||
|
return string(d.Distro)
|
||||||
|
}
|
||||||
|
|
||||||
|
func displayValue(v string) string {
|
||||||
|
if v == "" {
|
||||||
|
return "unknown"
|
||||||
|
}
|
||||||
|
return v
|
||||||
|
}
|
||||||
120
internal/sshinstall/detect_test.go
Обычный файл
120
internal/sshinstall/detect_test.go
Обычный файл
@@ -0,0 +1,120 @@
|
|||||||
|
package sshinstall
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
// prober returns a FileProber backed by a fixed path set.
|
||||||
|
func prober(exist ...string) FileProber {
|
||||||
|
want := make(map[string]bool, len(exist))
|
||||||
|
for _, p := range exist {
|
||||||
|
want[p] = true
|
||||||
|
}
|
||||||
|
return func(paths ...string) map[string]bool {
|
||||||
|
out := make(map[string]bool, len(paths))
|
||||||
|
for _, p := range paths {
|
||||||
|
out[p] = want[p]
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseOSReleaseQuoting(t *testing.T) {
|
||||||
|
d := parseOSRelease(`NAME="Ubuntu 24.04 LTS"
|
||||||
|
VERSION_ID="24.04"
|
||||||
|
ID=ubuntu
|
||||||
|
ID_LIKE=debian
|
||||||
|
PRETTY_NAME="Ubuntu 24.04 LTS"
|
||||||
|
`)
|
||||||
|
if d.ID != "ubuntu" || d.Name != "Ubuntu 24.04 LTS" || d.VersionID != "24.04" {
|
||||||
|
t.Fatalf("parseOSRelease = %+v", d)
|
||||||
|
}
|
||||||
|
if d.Distro != DistroUbuntu {
|
||||||
|
t.Fatalf("distro = %q, want ubuntu", d.Distro)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseOSReleaseSingleQuotedAndEmpty(t *testing.T) {
|
||||||
|
d := parseOSRelease("# comment\nID='alpine'\nNAME=Alpine\n\n")
|
||||||
|
if d.ID != "alpine" || d.Name != "Alpine" {
|
||||||
|
t.Fatalf("parseOSRelease = %+v", d)
|
||||||
|
}
|
||||||
|
if d.Distro != DistroAlpine {
|
||||||
|
t.Fatalf("distro = %q, want alpine", d.Distro)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseOSReleaseGarbage(t *testing.T) {
|
||||||
|
d := parseOSRelease("not an os-release file\nID = spaced\nFOO=bar\n")
|
||||||
|
if d.ID != "" {
|
||||||
|
t.Fatalf("ID = %q, want empty", d.ID)
|
||||||
|
}
|
||||||
|
if d.Distro != DistroUnknown {
|
||||||
|
t.Fatalf("distro = %q, want unknown", d.Distro)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPackageManagerFor(t *testing.T) {
|
||||||
|
cases := map[string]PackageManager{
|
||||||
|
"alpine": PkgApk,
|
||||||
|
"debian": PkgApt,
|
||||||
|
"ubuntu": PkgApt,
|
||||||
|
"arch": PkgPacman,
|
||||||
|
"centos": PkgDnf,
|
||||||
|
"fedora": PkgDnf,
|
||||||
|
"rocky": PkgDnf,
|
||||||
|
"almalinux": PkgDnf,
|
||||||
|
"rhel": PkgDnf,
|
||||||
|
"nonsense": PkgUnknown,
|
||||||
|
"": PkgUnknown,
|
||||||
|
}
|
||||||
|
for id, want := range cases {
|
||||||
|
if got := packageManagerFor(id); got != want {
|
||||||
|
t.Fatalf("packageManagerFor(%q) = %q, want %q", id, got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDetectInitMarkers(t *testing.T) {
|
||||||
|
osRelease := "ID=arch\nNAME=Arch Linux\n"
|
||||||
|
|
||||||
|
if got := Detect(osRelease, prober("/usr/lib/systemd/system")); got.InitSystem != InitSystemd {
|
||||||
|
t.Fatalf("arch systemd markers not detected: %+v", got)
|
||||||
|
}
|
||||||
|
if got := Detect(osRelease, nil).InitSystem; got != InitUnknown {
|
||||||
|
t.Fatalf("nil prober should yield unknown init, got %q", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
alpine := Detect("ID=alpine\n", prober("/sbin/openrc", "/etc/init.d"))
|
||||||
|
if alpine.InitSystem != InitOpenRC {
|
||||||
|
t.Fatalf("alpine openrc markers not detected: %+v", alpine)
|
||||||
|
}
|
||||||
|
if alpine.PackageManager != PkgApk {
|
||||||
|
t.Fatalf("alpine pkg = %q, want apk", alpine.PackageManager)
|
||||||
|
}
|
||||||
|
|
||||||
|
ubuntu := Detect("ID=ubuntu\n", prober("/run/systemd/system"))
|
||||||
|
if ubuntu.InitSystem != InitSystemd || ubuntu.PackageManager != PkgApt {
|
||||||
|
t.Fatalf("ubuntu detection = %+v", ubuntu)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDetectSystemdWinsOverOpenRC(t *testing.T) {
|
||||||
|
// A host that carries both markers must be reported as systemd so a
|
||||||
|
// systemd distro running a containerized openrc stub is not misplanned.
|
||||||
|
d := Detect("ID=ubuntu\n", prober("/usr/lib/systemd/system", "/sbin/openrc"))
|
||||||
|
if d.InitSystem != InitSystemd {
|
||||||
|
t.Fatalf("init = %q, want systemd", d.InitSystem)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDetectionSummarize(t *testing.T) {
|
||||||
|
d := Detect("ID=alpine\nNAME=Alpine Linux\n", prober("/sbin/openrc"))
|
||||||
|
s := d.Summarize()
|
||||||
|
for _, want := range []string{"Alpine Linux", "id=alpine", "pkg=apk", "init=openrc"} {
|
||||||
|
if !strings.Contains(s, want) {
|
||||||
|
t.Fatalf("Summarize() = %q missing %q", s, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
298
internal/sshinstall/plan.go
Обычный файл
298
internal/sshinstall/plan.go
Обычный файл
@@ -0,0 +1,298 @@
|
|||||||
|
package sshinstall
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"net/url"
|
||||||
|
"regexp"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
// DefaultGoVersion is the pinned Go toolchain version the source
|
||||||
|
// installer plans by default. It is overridable through install options
|
||||||
|
// when a host needs a different toolchain.
|
||||||
|
const DefaultGoVersion = "1.26.0"
|
||||||
|
|
||||||
|
// DefaultRepo is the publicly readable worker repository. It requires no
|
||||||
|
// source credential and is the plan's default clone URL.
|
||||||
|
const DefaultRepo = "https://rocketgit.ru/rsmon/worker.git"
|
||||||
|
|
||||||
|
var (
|
||||||
|
// goVersionPattern bounds Go toolchain version strings that are
|
||||||
|
// interpolated into remote shell commands and download URLs.
|
||||||
|
goVersionPattern = regexp.MustCompile(`^[0-9][0-9A-Za-z._-]*$`)
|
||||||
|
// goArchPattern bounds Go download archive suffixes (e.g. "amd64")
|
||||||
|
// that are interpolated into remote shell commands and URLs.
|
||||||
|
goArchPattern = regexp.MustCompile(`^[0-9A-Za-z][0-9A-Za-z_-]*$`)
|
||||||
|
)
|
||||||
|
|
||||||
|
// ValidGoVersion reports whether a Go toolchain version uses only safe
|
||||||
|
// characters (digits, letters, dots, dashes, underscores) and starts
|
||||||
|
// with a digit. Versions are embedded in remote shell commands and
|
||||||
|
// download URLs, so the charset is the injection boundary.
|
||||||
|
func ValidGoVersion(v string) bool { return goVersionPattern.MatchString(v) }
|
||||||
|
|
||||||
|
// ValidGoArch reports whether a Go download archive suffix uses only
|
||||||
|
// safe characters. Suffixes are embedded in remote shell commands and
|
||||||
|
// download URLs, so the charset is the injection boundary.
|
||||||
|
func ValidGoArch(a string) bool { return goArchPattern.MatchString(a) }
|
||||||
|
|
||||||
|
// Toolchain is a pinned, checksum-verified Go toolchain download for a
|
||||||
|
// remote Linux architecture. The SHA-256 is baked for the default
|
||||||
|
// toolchain version so planning never needs the network.
|
||||||
|
type Toolchain struct {
|
||||||
|
Version string // e.g. "1.26.0"
|
||||||
|
Arch string // download archive suffix, e.g. "linux-amd64"
|
||||||
|
URL string // direct download URL
|
||||||
|
SHA256 string // published SHA-256 (64 lowercase hex)
|
||||||
|
}
|
||||||
|
|
||||||
|
// toolchainSHA pins the official go.dev SHA-256 checksums for the
|
||||||
|
// default Go version per Linux archive. Sources: https://go.dev/dl/
|
||||||
|
// (?mode=json) published sums for DefaultGoVersion. Keep this in sync
|
||||||
|
// with DefaultGoVersion.
|
||||||
|
var toolchainSHA = map[string]string{
|
||||||
|
"linux-386": "35e2ec7a7ae6905a1fae5459197b70e3fcbc5e0a786a7d6ba8e49bcd38ad2e26",
|
||||||
|
"linux-amd64": "aac1b08a0fb0c4e0a7c1555beb7b59180b05dfc5a3d62e40e9de90cd42f88235",
|
||||||
|
"linux-arm64": "bd03b743eb6eb4193ea3c3fd3956546bf0e3ca5b7076c8226334afe6b75704cd",
|
||||||
|
"linux-armv6l": "3f6b48d96f0d8dff77e4625aa179e0449f6bbe79b6986bfa711c2cfc1257ebd8",
|
||||||
|
"linux-loong64": "33947cd7686f1cd5f097d2a5a30427a4ade114ea00b7570c85a2abf1af3d0507",
|
||||||
|
"linux-mips": "a4ece61d4bac43b6983fde2c6b9cfc1af7f0d5d6a073219583d4e93b11559c25",
|
||||||
|
"linux-mips64": "197c2e97fa9ec1ad05998e0982d1a1ae761980df154424e5f29f3912e9ea4e5e",
|
||||||
|
"linux-mips64le": "61c52b4ab0dceae29f10df29045483596c3f06810c9b511e8336a97428a95a1b",
|
||||||
|
"linux-mipsle": "b3a13cc5a5f9250b02cf4ba19914c90c7034e68a5ccb9affa5198aadbcedac9a",
|
||||||
|
"linux-ppc64": "ef7232a49101d163a93bac34d03bfbc4fb18f75d7526d77ac307e16d9d83c300",
|
||||||
|
"linux-ppc64le": "3066b2284b554da76cf664d217490792ba6f292ec0fc20bf9615e173cc0d2800",
|
||||||
|
"linux-riscv64": "ab9226ecddda0f682365c949114b653a66c2e9330e7b8d3edea80858437d2ff2",
|
||||||
|
"linux-s390x": "d62137f11530b97f3503453ad7d9e570af070770599fb8054f4e8cd0e905a453",
|
||||||
|
}
|
||||||
|
|
||||||
|
// GoArch maps a remote `uname -m` value to the Go download archive
|
||||||
|
// suffix used by go.dev. Every suffix it can return must have a pinned
|
||||||
|
// checksum in toolchainSHA; the table-consistency test enforces that.
|
||||||
|
// Unknown values error.
|
||||||
|
func GoArch(unameM string) (string, error) {
|
||||||
|
switch strings.ToLower(strings.TrimSpace(unameM)) {
|
||||||
|
case "x86_64", "amd64":
|
||||||
|
return "amd64", nil
|
||||||
|
case "aarch64", "arm64":
|
||||||
|
return "arm64", nil
|
||||||
|
case "armv6l", "armv7l":
|
||||||
|
return "armv6l", nil
|
||||||
|
case "i386", "i486", "i586", "i686", "386":
|
||||||
|
return "386", nil
|
||||||
|
case "loongarch64":
|
||||||
|
return "loong64", nil
|
||||||
|
case "mips":
|
||||||
|
return "mips", nil
|
||||||
|
case "mipsel":
|
||||||
|
return "mipsle", nil
|
||||||
|
case "mips64":
|
||||||
|
return "mips64", nil
|
||||||
|
case "mips64el":
|
||||||
|
return "mips64le", nil
|
||||||
|
case "ppc64":
|
||||||
|
return "ppc64", nil
|
||||||
|
case "ppc64le":
|
||||||
|
return "ppc64le", nil
|
||||||
|
case "riscv64":
|
||||||
|
return "riscv64", nil
|
||||||
|
case "s390x":
|
||||||
|
return "s390x", nil
|
||||||
|
default:
|
||||||
|
return "", fmt.Errorf("unsupported machine architecture %q", unameM)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ToolchainFor returns the pinned, checksum-verified Go toolchain for a
|
||||||
|
// remote Linux architecture. A non-default version has no baked
|
||||||
|
// checksum yet and must be resolved through a checksum source by the
|
||||||
|
// executor work package.
|
||||||
|
func ToolchainFor(goarch, version string) (Toolchain, error) {
|
||||||
|
if version == "" {
|
||||||
|
version = DefaultGoVersion
|
||||||
|
}
|
||||||
|
suffix := "linux-" + goarch
|
||||||
|
t := Toolchain{
|
||||||
|
Version: version,
|
||||||
|
Arch: suffix,
|
||||||
|
URL: "https://go.dev/dl/go" + version + "." + suffix + ".tar.gz",
|
||||||
|
}
|
||||||
|
if version != DefaultGoVersion {
|
||||||
|
return t, fmt.Errorf("no baked checksum for Go %s; only %s is pinned (resolve %s via the checksum source)",
|
||||||
|
version, DefaultGoVersion, suffix)
|
||||||
|
}
|
||||||
|
sha, ok := toolchainSHA[suffix]
|
||||||
|
if !ok {
|
||||||
|
return Toolchain{}, fmt.Errorf("no pinned Go %s toolchain for %s", version, suffix)
|
||||||
|
}
|
||||||
|
t.SHA256 = sha
|
||||||
|
return t, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// SourceOptions are the operator-configurable knobs that the source
|
||||||
|
// plan is resolved against. Every field is optional; empty values fall
|
||||||
|
// back to the pinned defaults.
|
||||||
|
type SourceOptions struct {
|
||||||
|
Repo string // clone URL; default DefaultRepo
|
||||||
|
Branch string // branch to build; empty means "the remote default branch"
|
||||||
|
GoVersion string // default DefaultGoVersion
|
||||||
|
GoArch string // go archive suffix; when empty, derived from UnameM
|
||||||
|
UnameM string // remote `uname -m` output; required unless GoArch set
|
||||||
|
BuildDir string // remote clone/build directory
|
||||||
|
GoModuleProxy string // GOPROXY override; empty keeps the Go default
|
||||||
|
}
|
||||||
|
|
||||||
|
// SourcePlan is the pure, resolved source-install plan. Producing it
|
||||||
|
// never touches the network or the remote host; the executor work
|
||||||
|
// package turns it into remote commands.
|
||||||
|
type SourcePlan struct {
|
||||||
|
Repo string
|
||||||
|
Branch string
|
||||||
|
BuildDir string
|
||||||
|
GoModuleProxy string
|
||||||
|
Toolchain Toolchain
|
||||||
|
Packages []string // packages to install via the package manager
|
||||||
|
InitSystem InitSystem
|
||||||
|
}
|
||||||
|
|
||||||
|
// PlanSource resolves a source-install plan for a detected host. It
|
||||||
|
// returns an error before any remote mutation could happen when the
|
||||||
|
// target cannot be planned (unknown distro, unsupported architecture,
|
||||||
|
// malformed repository).
|
||||||
|
func PlanSource(d Detection, opts SourceOptions) (SourcePlan, error) {
|
||||||
|
if d.PackageManager == PkgUnknown {
|
||||||
|
return SourcePlan{}, fmt.Errorf("unsupported distro %q: no package manager", d.ID)
|
||||||
|
}
|
||||||
|
goarch := strings.TrimSpace(opts.GoArch)
|
||||||
|
if goarch != "" && !ValidGoArch(goarch) {
|
||||||
|
return SourcePlan{}, fmt.Errorf("invalid Go architecture %q: only letters, digits, dashes, and underscores are allowed", goarch)
|
||||||
|
}
|
||||||
|
if goarch == "" {
|
||||||
|
var err error
|
||||||
|
goarch, err = GoArch(opts.UnameM)
|
||||||
|
if err != nil {
|
||||||
|
return SourcePlan{}, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if version := strings.TrimSpace(opts.GoVersion); version != "" && !ValidGoVersion(version) {
|
||||||
|
return SourcePlan{}, fmt.Errorf("invalid Go version %q: only digits, letters, dots, dashes, and underscores are allowed", version)
|
||||||
|
}
|
||||||
|
toolchain, err := ToolchainFor(goarch, opts.GoVersion)
|
||||||
|
if err != nil {
|
||||||
|
return SourcePlan{}, err
|
||||||
|
}
|
||||||
|
repo := strings.TrimSpace(opts.Repo)
|
||||||
|
if repo == "" {
|
||||||
|
repo = DefaultRepo
|
||||||
|
}
|
||||||
|
if err := validateRepoURL(repo); err != nil {
|
||||||
|
return SourcePlan{}, err
|
||||||
|
}
|
||||||
|
branch := strings.TrimSpace(opts.Branch)
|
||||||
|
// An empty branch means "build the remote's default branch" (the
|
||||||
|
// public repo currently publishes master). The executor resolves and
|
||||||
|
// records the remote default; a non-empty branch is pinned and must
|
||||||
|
// exist on the remote.
|
||||||
|
buildDir := strings.TrimSpace(opts.BuildDir)
|
||||||
|
if buildDir == "" {
|
||||||
|
buildDir = "/opt/rsmon-worker-src"
|
||||||
|
}
|
||||||
|
if !strings.HasPrefix(buildDir, "/") {
|
||||||
|
return SourcePlan{}, fmt.Errorf("build directory must be absolute, got %q", buildDir)
|
||||||
|
}
|
||||||
|
return SourcePlan{
|
||||||
|
Repo: repo,
|
||||||
|
Branch: branch,
|
||||||
|
BuildDir: buildDir,
|
||||||
|
GoModuleProxy: strings.TrimSpace(opts.GoModuleProxy),
|
||||||
|
Toolchain: toolchain,
|
||||||
|
Packages: packagePrereqs(d.PackageManager),
|
||||||
|
InitSystem: d.InitSystem,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// packagePrereqs returns the minimal package set the source installer
|
||||||
|
// plans to install for a package manager: git, CA certificates, and
|
||||||
|
// download/archive tools. It never plans a C compiler or build-essential
|
||||||
|
// because the worker builds with CGO disabled.
|
||||||
|
func packagePrereqs(pkg PackageManager) []string {
|
||||||
|
switch pkg {
|
||||||
|
case PkgApk:
|
||||||
|
return []string{"git", "ca-certificates", "curl", "tar", "gzip"}
|
||||||
|
case PkgApt:
|
||||||
|
return []string{"git", "ca-certificates", "curl", "tar", "gzip"}
|
||||||
|
case PkgPacman:
|
||||||
|
return []string{"git", "ca-certificates", "curl", "tar", "gzip"}
|
||||||
|
case PkgDnf, PkgYum:
|
||||||
|
return []string{"git", "ca-certificates", "curl", "tar", "gzip"}
|
||||||
|
default:
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ValidateRepoURL rejects repository references that could smuggle a
|
||||||
|
// command or a non-remote scheme into the clone step. Only https is
|
||||||
|
// accepted (the default repository is https), and userinfo such as
|
||||||
|
// `user:pass@host` is rejected so credentials can never reach the remote
|
||||||
|
// clone command or the clone's config. An empty value is accepted here
|
||||||
|
// (it falls back to the default repository when planning).
|
||||||
|
func ValidateRepoURL(repo string) error {
|
||||||
|
if repo == "" {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return validateRepoURL(repo)
|
||||||
|
}
|
||||||
|
|
||||||
|
func validateRepoURL(repo string) error {
|
||||||
|
if strings.ContainsAny(repo, "\r\n\t ") {
|
||||||
|
return fmt.Errorf("repository URL %q contains whitespace", repo)
|
||||||
|
}
|
||||||
|
u, err := url.Parse(repo)
|
||||||
|
if err != nil || u.Host == "" {
|
||||||
|
return fmt.Errorf("repository URL %q is not an absolute clone URL", repo)
|
||||||
|
}
|
||||||
|
if u.User != nil {
|
||||||
|
return fmt.Errorf("repository URL %q must not contain userinfo", repo)
|
||||||
|
}
|
||||||
|
if u.Scheme != "https" {
|
||||||
|
return fmt.Errorf("repository URL %q must use the https scheme", repo)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// StepKind identifies one ordered remote step the executor will run.
|
||||||
|
type StepKind string
|
||||||
|
|
||||||
|
const (
|
||||||
|
StepInstallPackages StepKind = "install-packages"
|
||||||
|
StepInstallToolchain StepKind = "install-toolchain"
|
||||||
|
StepCloneSource StepKind = "clone-source"
|
||||||
|
StepCheckoutBranch StepKind = "checkout-branch"
|
||||||
|
StepBuildWorker StepKind = "build-worker"
|
||||||
|
StepInstallService StepKind = "install-service"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Step is one ordered, pure planning step. The executor maps each step
|
||||||
|
// to remote commands; planning does not execute anything.
|
||||||
|
type Step struct {
|
||||||
|
Kind StepKind
|
||||||
|
Detail string
|
||||||
|
Packages []string // only for StepInstallPackages
|
||||||
|
}
|
||||||
|
|
||||||
|
// Steps returns the ordered source-install plan as stable, reviewable
|
||||||
|
// steps. It is the contract the executor work package implements.
|
||||||
|
func (p SourcePlan) Steps() []Step {
|
||||||
|
checkoutDetail := "check out branch " + p.Branch + " and record the resolved commit"
|
||||||
|
if p.Branch == "" {
|
||||||
|
checkoutDetail = "check out the remote default branch and record the resolved commit"
|
||||||
|
}
|
||||||
|
return []Step{
|
||||||
|
{Kind: StepInstallPackages, Detail: "install minimal build prerequisites", Packages: p.Packages},
|
||||||
|
{Kind: StepInstallToolchain, Detail: "install pinned Go " + p.Toolchain.Version + " (" + p.Toolchain.Arch + ") and verify SHA-256"},
|
||||||
|
{Kind: StepCloneSource, Detail: "clone " + p.Repo + " into " + p.BuildDir},
|
||||||
|
{Kind: StepCheckoutBranch, Detail: checkoutDetail},
|
||||||
|
{Kind: StepBuildWorker, Detail: "build the worker binary with CGO_ENABLED=0 and trimpath"},
|
||||||
|
{Kind: StepInstallService, Detail: "atomically install the binary, env, data dir, and " + string(p.InitSystem) + " service definition"},
|
||||||
|
}
|
||||||
|
}
|
||||||
274
internal/sshinstall/plan_test.go
Обычный файл
274
internal/sshinstall/plan_test.go
Обычный файл
@@ -0,0 +1,274 @@
|
|||||||
|
package sshinstall
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestGoArch(t *testing.T) {
|
||||||
|
cases := map[string]string{
|
||||||
|
"x86_64": "amd64",
|
||||||
|
"X86_64": "amd64",
|
||||||
|
"amd64": "amd64",
|
||||||
|
"aarch64": "arm64",
|
||||||
|
"arm64": "arm64",
|
||||||
|
"armv7l": "armv6l",
|
||||||
|
"armv6l": "armv6l",
|
||||||
|
"i686": "386",
|
||||||
|
"i386": "386",
|
||||||
|
"loongarch64": "loong64",
|
||||||
|
"mips": "mips",
|
||||||
|
"mipsel": "mipsle",
|
||||||
|
"mips64": "mips64",
|
||||||
|
"mips64el": "mips64le",
|
||||||
|
"ppc64": "ppc64",
|
||||||
|
"ppc64le": "ppc64le",
|
||||||
|
"riscv64": "riscv64",
|
||||||
|
"s390x": "s390x",
|
||||||
|
}
|
||||||
|
for in, want := range cases {
|
||||||
|
got, err := GoArch(in)
|
||||||
|
if err != nil || got != want {
|
||||||
|
t.Fatalf("GoArch(%q) = %q, %v; want %q", in, got, err, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, bad := range []string{"", "sparc", "x86", "mips64el-le"} {
|
||||||
|
if _, err := GoArch(bad); err == nil {
|
||||||
|
t.Fatalf("GoArch(%q) succeeded", bad)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestToolchainTableConsistentWithGoArch keeps GoArch and toolchainSHA in
|
||||||
|
// sync: every `uname -m` mapping must resolve to a pinned checksum, and
|
||||||
|
// every pinned checksum must be reachable through some `uname -m` value.
|
||||||
|
func TestToolchainTableConsistentWithGoArch(t *testing.T) {
|
||||||
|
// unameExamples maps a Go archive suffix to a real `uname -m` value
|
||||||
|
// that GoArch accepts for it.
|
||||||
|
unameExamples := map[string]string{
|
||||||
|
"amd64": "x86_64",
|
||||||
|
"arm64": "aarch64",
|
||||||
|
"armv6l": "armv7l",
|
||||||
|
"386": "i686",
|
||||||
|
"loong64": "loongarch64",
|
||||||
|
"mips": "mips",
|
||||||
|
"mipsle": "mipsel",
|
||||||
|
"mips64": "mips64",
|
||||||
|
"mips64le": "mips64el",
|
||||||
|
"ppc64": "ppc64",
|
||||||
|
"ppc64le": "ppc64le",
|
||||||
|
"riscv64": "riscv64",
|
||||||
|
"s390x": "s390x",
|
||||||
|
}
|
||||||
|
for archSuffix, unameValue := range unameExamples {
|
||||||
|
tc, err := ToolchainFor(archSuffix, DefaultGoVersion)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("GoArch(%q) = %q has no pinned checksum: %v", unameValue, archSuffix, err)
|
||||||
|
}
|
||||||
|
if tc.SHA256 == "" || tc.Arch != "linux-"+archSuffix {
|
||||||
|
t.Fatalf("ToolchainFor(%q) = %+v", archSuffix, tc)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for suffix := range toolchainSHA {
|
||||||
|
arch := strings.TrimPrefix(suffix, "linux-")
|
||||||
|
if _, ok := unameExamples[arch]; !ok {
|
||||||
|
t.Fatalf("checksum %q is not reachable through any GoArch `uname -m` mapping", suffix)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestToolchainFor(t *testing.T) {
|
||||||
|
tc, err := ToolchainFor("amd64", "")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if tc.Version != DefaultGoVersion {
|
||||||
|
t.Fatalf("version = %q, want %q", tc.Version, DefaultGoVersion)
|
||||||
|
}
|
||||||
|
if tc.Arch != "linux-amd64" {
|
||||||
|
t.Fatalf("arch = %q, want linux-amd64", tc.Arch)
|
||||||
|
}
|
||||||
|
if tc.URL != "https://go.dev/dl/go1.26.0.linux-amd64.tar.gz" {
|
||||||
|
t.Fatalf("url = %q", tc.URL)
|
||||||
|
}
|
||||||
|
if len(tc.SHA256) != 64 {
|
||||||
|
t.Fatalf("sha = %q, want 64 hex chars", tc.SHA256)
|
||||||
|
}
|
||||||
|
for _, c := range tc.SHA256 {
|
||||||
|
hexDigit := c >= '0' && c <= '9' || c >= 'a' && c <= 'f'
|
||||||
|
if !hexDigit {
|
||||||
|
t.Fatalf("sha %q contains non-lowercase-hex char", tc.SHA256)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestToolchainForArm64Pinned(t *testing.T) {
|
||||||
|
tc, err := ToolchainFor("arm64", DefaultGoVersion)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if tc.SHA256 == "" || tc.Arch != "linux-arm64" {
|
||||||
|
t.Fatalf("arm64 toolchain not pinned: %+v", tc)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestToolchainForRejectsUnpinnedVersion(t *testing.T) {
|
||||||
|
if _, err := ToolchainFor("amd64", "1.27.0"); err == nil {
|
||||||
|
t.Fatal("unpinned Go version accepted")
|
||||||
|
}
|
||||||
|
if _, err := ToolchainFor("sparc", ""); err == nil {
|
||||||
|
t.Fatal("unknown architecture accepted")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPlanSourceAlpine(t *testing.T) {
|
||||||
|
d := Detect("ID=alpine\n", prober("/sbin/openrc"))
|
||||||
|
p, err := PlanSource(d, SourceOptions{UnameM: "x86_64"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if p.Repo != DefaultRepo || p.Branch != "" {
|
||||||
|
t.Fatalf("plan defaults wrong: %+v", p)
|
||||||
|
}
|
||||||
|
if p.Toolchain.Arch != "linux-amd64" {
|
||||||
|
t.Fatalf("toolchain = %+v", p.Toolchain)
|
||||||
|
}
|
||||||
|
if len(p.Packages) != 5 || p.Packages[0] != "git" {
|
||||||
|
t.Fatalf("alpine packages = %v", p.Packages)
|
||||||
|
}
|
||||||
|
if p.InitSystem != InitOpenRC {
|
||||||
|
t.Fatalf("init = %q, want openrc", p.InitSystem)
|
||||||
|
}
|
||||||
|
steps := p.Steps()
|
||||||
|
if len(steps) != 6 {
|
||||||
|
t.Fatalf("steps = %d, want 6", len(steps))
|
||||||
|
}
|
||||||
|
if steps[0].Kind != StepInstallPackages || steps[1].Kind != StepInstallToolchain {
|
||||||
|
t.Fatalf("step order wrong: %+v", steps)
|
||||||
|
}
|
||||||
|
if !strings.Contains(steps[1].Detail, "Go 1.26.0") || !strings.Contains(steps[1].Detail, "linux-amd64") {
|
||||||
|
t.Fatalf("toolchain step detail = %q", steps[1].Detail)
|
||||||
|
}
|
||||||
|
if !strings.Contains(steps[5].Detail, "openrc") {
|
||||||
|
t.Fatalf("install step detail = %q", steps[5].Detail)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPlanSourceUbuntuArch(t *testing.T) {
|
||||||
|
ubuntu := Detect("ID=ubuntu\n", prober("/usr/lib/systemd/system"))
|
||||||
|
p, err := PlanSource(ubuntu, SourceOptions{UnameM: "aarch64"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if p.Toolchain.Arch != "linux-arm64" || p.InitSystem != InitSystemd {
|
||||||
|
t.Fatalf("ubuntu plan = %+v", p)
|
||||||
|
}
|
||||||
|
|
||||||
|
arch := Detect("ID=arch\n", prober("/usr/lib/systemd/system"))
|
||||||
|
p, err = PlanSource(arch, SourceOptions{UnameM: "x86_64"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if p.Packages[0] != "git" || p.InitSystem != InitSystemd {
|
||||||
|
t.Fatalf("arch plan = %+v", p)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPlanSourceExplicitOverrides(t *testing.T) {
|
||||||
|
d := Detect("ID=ubuntu\n", nil)
|
||||||
|
p, err := PlanSource(d, SourceOptions{
|
||||||
|
Repo: "https://example.test/worker.git",
|
||||||
|
Branch: "release-1.0",
|
||||||
|
GoArch: "arm64",
|
||||||
|
BuildDir: "/srv/rsmon",
|
||||||
|
GoModuleProxy: "https://proxy.golang.org,direct",
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if p.Repo != "https://example.test/worker.git" || p.Branch != "release-1.0" ||
|
||||||
|
p.BuildDir != "/srv/rsmon" || p.GoModuleProxy != "https://proxy.golang.org,direct" {
|
||||||
|
t.Fatalf("overrides not applied: %+v", p)
|
||||||
|
}
|
||||||
|
if p.Toolchain.Arch != "linux-arm64" {
|
||||||
|
t.Fatalf("toolchain = %+v", p.Toolchain)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPlanSourceErrors(t *testing.T) {
|
||||||
|
unknown := Detect("ID=weirdos\n", nil)
|
||||||
|
if _, err := PlanSource(unknown, SourceOptions{UnameM: "x86_64"}); err == nil {
|
||||||
|
t.Fatal("unknown distro planned")
|
||||||
|
}
|
||||||
|
|
||||||
|
ubuntu := Detect("ID=ubuntu\n", nil)
|
||||||
|
if _, err := PlanSource(ubuntu, SourceOptions{}); err == nil {
|
||||||
|
t.Fatal("missing architecture planned")
|
||||||
|
}
|
||||||
|
for _, repo := range []string{"file:///tmp/worker.git", "not-a-url", "https://ex ample/x"} {
|
||||||
|
if _, err := PlanSource(ubuntu, SourceOptions{UnameM: "x86_64", Repo: repo}); err == nil {
|
||||||
|
t.Fatalf("invalid repo %q planned", repo)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if _, err := PlanSource(ubuntu, SourceOptions{UnameM: "x86_64", BuildDir: "relative"}); err == nil {
|
||||||
|
t.Fatal("relative build dir planned")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPackagePrereqsNeverIncludeCompiler(t *testing.T) {
|
||||||
|
for _, pkg := range []PackageManager{PkgApk, PkgApt, PkgPacman, PkgDnf} {
|
||||||
|
for _, name := range packagePrereqs(pkg) {
|
||||||
|
switch name {
|
||||||
|
case "build-essential", "gcc", "g++", "base-devel", "gcc-c++", "make":
|
||||||
|
t.Fatalf("plan includes compiler package %q", name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if got := packagePrereqs(PkgUnknown); len(got) != 0 {
|
||||||
|
t.Fatalf("unknown package manager planned packages %v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestValidateRepoURLSchemes(t *testing.T) {
|
||||||
|
for _, ok := range []string{"https://rocketgit.ru/rsmon/worker.git", "https://example.test/r"} {
|
||||||
|
if err := validateRepoURL(ok); err != nil {
|
||||||
|
t.Fatalf("validateRepoURL(%q): %v", ok, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, bad := range []string{
|
||||||
|
"ssh://h@x/r", "s3://bucket/key", "x y", "", "http://x/y", "git://example.test/r",
|
||||||
|
"https://user:pass@example.test/r", "https://token@example.test/r", "file:///tmp/r",
|
||||||
|
} {
|
||||||
|
if err := validateRepoURL(bad); err == nil {
|
||||||
|
t.Fatalf("validateRepoURL(%q) succeeded", bad)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPlanSourceRejectsUnsafeCharset(t *testing.T) {
|
||||||
|
ubuntu := Detect("ID=ubuntu\n", nil)
|
||||||
|
for _, goarch := range []string{"amd64;rm", "x;rm -rf", "$(id)", "..", "a b"} {
|
||||||
|
if _, err := PlanSource(ubuntu, SourceOptions{UnameM: "x86_64", GoArch: goarch}); err == nil {
|
||||||
|
t.Fatalf("unsafe GoArch %q planned", goarch)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, version := range []string{"1.26;rm", "$(id)", "1.26.0 x", "a/b"} {
|
||||||
|
if _, err := PlanSource(ubuntu, SourceOptions{UnameM: "x86_64", GoVersion: version}); err == nil {
|
||||||
|
t.Fatalf("unsafe GoVersion %q planned", version)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !ValidGoVersion("1.26.0") || !ValidGoArch("amd64") {
|
||||||
|
t.Fatal("valid version/arch rejected")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPlanSourceEmptyBranchSteps(t *testing.T) {
|
||||||
|
ubuntu := Detect("ID=ubuntu\n", nil)
|
||||||
|
p, err := PlanSource(ubuntu, SourceOptions{UnameM: "x86_64"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !strings.Contains(p.Steps()[3].Detail, "remote default branch") {
|
||||||
|
t.Fatalf("checkout step detail for empty branch = %q", p.Steps()[3].Detail)
|
||||||
|
}
|
||||||
|
}
|
||||||
176
internal/webapp/compose.go
Обычный файл
176
internal/webapp/compose.go
Обычный файл
@@ -0,0 +1,176 @@
|
|||||||
|
package webapp
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"log"
|
||||||
|
"os"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"rocketgit.ru/rsmon/worker/internal/compose"
|
||||||
|
)
|
||||||
|
|
||||||
|
// composeRefreshInterval is the cadence the Compose project snapshot is
|
||||||
|
// rebuilt. 60s matches the process-inventory loop so the two collectors
|
||||||
|
// stay in step; management operations are synchronous POSTs that do not
|
||||||
|
// wait for the next tick.
|
||||||
|
const composeRefreshInterval = 60 * time.Second
|
||||||
|
|
||||||
|
// ComposeRefresher owns the periodic `docker compose` discovery loop and
|
||||||
|
// exposes the latest snapshot for the operator console. It is the
|
||||||
|
// Compose counterpart of Inventory: a self-contained refresher that is
|
||||||
|
// constructed in New and started in Start.
|
||||||
|
//
|
||||||
|
// When Docker is unavailable (or compose is disabled) the refresher
|
||||||
|
// stays idle and reports an empty snapshot with an "unavailable" error,
|
||||||
|
// so /compose renders a banner instead of a half-built table.
|
||||||
|
type ComposeRefresher struct {
|
||||||
|
enabled bool
|
||||||
|
log *log.Logger
|
||||||
|
disco composeDiscover
|
||||||
|
|
||||||
|
mu sync.RWMutex
|
||||||
|
snap compose.Snapshot
|
||||||
|
lastAt time.Time
|
||||||
|
stopCh chan struct{}
|
||||||
|
stopWG sync.WaitGroup
|
||||||
|
started bool
|
||||||
|
}
|
||||||
|
|
||||||
|
// composeDiscover is the subset of the compose package the refresher
|
||||||
|
// calls. A function field so tests can inject a fixture discovery
|
||||||
|
// without exec'ing Docker.
|
||||||
|
type composeDiscover func(ctx context.Context) (*compose.DiscoveryResult, error)
|
||||||
|
|
||||||
|
// NewComposeRefresher constructs a refresher. When enabled is false the
|
||||||
|
// refresher never starts its loop and Snapshot returns an empty result;
|
||||||
|
// the page still renders (with the disabled notice). The discover hook
|
||||||
|
// defaults to compose.Discover so production needs no wiring.
|
||||||
|
func NewComposeRefresher(enabled bool, logger *log.Logger) *ComposeRefresher {
|
||||||
|
if logger == nil {
|
||||||
|
logger = log.New(os.Stderr, "webapp-compose: ", log.LstdFlags)
|
||||||
|
}
|
||||||
|
return &ComposeRefresher{
|
||||||
|
enabled: enabled,
|
||||||
|
log: logger,
|
||||||
|
disco: compose.Discover,
|
||||||
|
stopCh: make(chan struct{}),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// SetDiscovery injects the discovery function. Used by tests; production
|
||||||
|
// leaves the compose.Discover default set by NewComposeRefresher.
|
||||||
|
func (c *ComposeRefresher) SetDiscovery(f composeDiscover) {
|
||||||
|
if f == nil {
|
||||||
|
c.disco = compose.Discover
|
||||||
|
return
|
||||||
|
}
|
||||||
|
c.disco = f
|
||||||
|
}
|
||||||
|
|
||||||
|
// Enabled reports whether compose management is turned on.
|
||||||
|
func (c *ComposeRefresher) Enabled() bool { return c != nil && c.enabled }
|
||||||
|
|
||||||
|
// Start launches the background refresh loop. The first refresh runs
|
||||||
|
// immediately so /compose has data on the first request. Idempotent.
|
||||||
|
func (c *ComposeRefresher) Start(ctx context.Context) {
|
||||||
|
if c == nil || !c.enabled {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
c.mu.Lock()
|
||||||
|
if c.started {
|
||||||
|
c.mu.Unlock()
|
||||||
|
return
|
||||||
|
}
|
||||||
|
c.started = true
|
||||||
|
c.mu.Unlock()
|
||||||
|
|
||||||
|
c.stopWG.Add(1)
|
||||||
|
go c.loop(ctx)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Stop cancels the refresh loop and waits for it to exit.
|
||||||
|
func (c *ComposeRefresher) Stop() {
|
||||||
|
if c == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
c.mu.Lock()
|
||||||
|
if !c.started {
|
||||||
|
c.mu.Unlock()
|
||||||
|
return
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case <-c.stopCh:
|
||||||
|
default:
|
||||||
|
close(c.stopCh)
|
||||||
|
}
|
||||||
|
c.mu.Unlock()
|
||||||
|
c.stopWG.Wait()
|
||||||
|
}
|
||||||
|
|
||||||
|
// Snapshot returns the most recent discovery snapshot. Always safe to
|
||||||
|
// call; returns a zero-value snapshot (with the unavailable notice when
|
||||||
|
// disabled) before the first refresh completes.
|
||||||
|
func (c *ComposeRefresher) Snapshot() compose.Snapshot {
|
||||||
|
if c == nil {
|
||||||
|
return compose.Snapshot{Errors: []string{"compose management is disabled"}}
|
||||||
|
}
|
||||||
|
c.mu.RLock()
|
||||||
|
defer c.mu.RUnlock()
|
||||||
|
if !c.enabled {
|
||||||
|
return compose.Snapshot{Errors: []string{"compose management is disabled"}}
|
||||||
|
}
|
||||||
|
out := c.snap
|
||||||
|
out.Projects = append([]compose.ProjectSummary(nil), c.snap.Projects...)
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// LastRefreshAt returns the time of the last successful refresh.
|
||||||
|
func (c *ComposeRefresher) LastRefreshAt() time.Time {
|
||||||
|
c.mu.RLock()
|
||||||
|
defer c.mu.RUnlock()
|
||||||
|
return c.lastAt
|
||||||
|
}
|
||||||
|
|
||||||
|
// Refs resolves the Compose CLI arguments for a project from the current
|
||||||
|
// snapshot. Returns ok=false when the project is unknown or its working
|
||||||
|
// directory was not discovered (Compose cannot be driven without it).
|
||||||
|
func (c *ComposeRefresher) Refs(project string) (compose.ProjectRefs, bool) {
|
||||||
|
p := c.Snapshot().FindProject(project)
|
||||||
|
if p == nil || p.WorkingDir == "" {
|
||||||
|
return compose.ProjectRefs{}, false
|
||||||
|
}
|
||||||
|
return compose.ProjectRefs{Name: p.Name, WorkingDir: p.WorkingDir, ConfigFiles: p.ConfigFiles}, true
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *ComposeRefresher) loop(ctx context.Context) {
|
||||||
|
defer c.stopWG.Done()
|
||||||
|
c.refresh(ctx)
|
||||||
|
t := time.NewTicker(composeRefreshInterval)
|
||||||
|
defer t.Stop()
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return
|
||||||
|
case <-c.stopCh:
|
||||||
|
return
|
||||||
|
case <-t.C:
|
||||||
|
c.refresh(ctx)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *ComposeRefresher) refresh(ctx context.Context) {
|
||||||
|
res, err := c.disco(ctx)
|
||||||
|
if err != nil || res == nil {
|
||||||
|
if err != nil {
|
||||||
|
c.log.Printf("compose refresh: %v", err)
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
snap := compose.Summarize(res)
|
||||||
|
c.mu.Lock()
|
||||||
|
c.snap = snap
|
||||||
|
c.lastAt = time.Now().UTC()
|
||||||
|
c.mu.Unlock()
|
||||||
|
}
|
||||||
@@ -30,7 +30,7 @@ const (
|
|||||||
envWorkerLogin = "WORKER_LOGIN"
|
envWorkerLogin = "WORKER_LOGIN"
|
||||||
envWorkerPassword = "WORKER_PASSWORD"
|
envWorkerPassword = "WORKER_PASSWORD"
|
||||||
envClusterEnabled = "WORKER_CLUSTER_ENABLED"
|
envClusterEnabled = "WORKER_CLUSTER_ENABLED"
|
||||||
envClusterDebugApply = "WORKER_CLUSTER_DEBUG_APPLY"
|
envComposeEnabled = "WORKER_COMPOSE_ENABLED"
|
||||||
envReleaseURL = "WORKER_RELEASE_URL"
|
envReleaseURL = "WORKER_RELEASE_URL"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
@@ -74,63 +74,6 @@ func (s *Server) handleClusterStatus(w http.ResponseWriter, _ *http.Request) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// handleClusterApplyTestConfig applies a hardcoded config.adopt log
|
|
||||||
// entry to the cluster. It exists so the e2e script and any operator
|
|
||||||
// debugging session can verify FSM replication without having to wire
|
|
||||||
// up the real signed-config-adoption producer (which lives in a later
|
|
||||||
// phase).
|
|
||||||
//
|
|
||||||
// DEBUG: this endpoint is a placeholder for the real producer. It must
|
|
||||||
// be replaced (or removed) before any production deployment.
|
|
||||||
//
|
|
||||||
// The handler is gated behind Config.DebugClusterApply (env
|
|
||||||
// WORKER_CLUSTER_DEBUG_APPLY=true). When the flag is false the
|
|
||||||
// handler returns 404 — the route is still registered so the auth
|
|
||||||
// + CSRF paths are exercised in tests, but no real FSM entry is ever
|
|
||||||
// appended from a production webapp.
|
|
||||||
//
|
|
||||||
// TODO(worker-cluster-real-producer): remove the apply-test-config
|
|
||||||
// endpoint entirely once the signed-config-adoption producer ships.
|
|
||||||
func (s *Server) handleClusterApplyTestConfig(w http.ResponseWriter, r *http.Request) {
|
|
||||||
writeNoStore(w)
|
|
||||||
if !s.cfg.DebugClusterApply {
|
|
||||||
http.NotFound(w, r)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if s.cluster == nil {
|
|
||||||
http.Error(w, "cluster not configured", http.StatusServiceUnavailable)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if !s.requireCSRF(sessionFromContextOrFail(w, r), r) {
|
|
||||||
http.Error(w, "csrf token required", http.StatusForbidden)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
applied, err := s.cluster.ApplyTestConfig()
|
|
||||||
if err != nil {
|
|
||||||
s.deps.Logger.Printf("cluster apply test config: %v", err)
|
|
||||||
http.Error(w, err.Error(), http.StatusBadGateway)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
w.Header().Set("Content-Type", "application/json; charset=utf-8")
|
|
||||||
w.WriteHeader(http.StatusOK)
|
|
||||||
if err := json.NewEncoder(w).Encode(map[string]uint64{"applied_index": applied}); err != nil {
|
|
||||||
s.deps.Logger.Printf("cluster apply encode: %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// sessionFromContextOrFail is a tiny adapter so requireCSRF can be
|
|
||||||
// called from this handler without leaking the middleware into the
|
|
||||||
// cluster package. If no session is attached (should not happen
|
|
||||||
// because requireSession already ran) we return a stub session with
|
|
||||||
// no CSRF token, which causes requireCSRF to refuse the request.
|
|
||||||
func sessionFromContextOrFail(_ http.ResponseWriter, r *http.Request) *Session {
|
|
||||||
sess, _ := sessionFromContext(r.Context())
|
|
||||||
if sess != nil {
|
|
||||||
return sess
|
|
||||||
}
|
|
||||||
return &Session{}
|
|
||||||
}
|
|
||||||
|
|
||||||
// ErrClusterNotConfigured is returned when a cluster-admin endpoint is
|
// ErrClusterNotConfigured is returned when a cluster-admin endpoint is
|
||||||
// hit on a server without a cluster attached.
|
// hit on a server without a cluster attached.
|
||||||
var ErrClusterNotConfigured = errors.New("webapp: cluster not configured")
|
var ErrClusterNotConfigured = errors.New("webapp: cluster not configured")
|
||||||
|
|||||||
@@ -1,16 +1,11 @@
|
|||||||
package webapp
|
package webapp
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"io"
|
"io"
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/http/httptest"
|
"net/http/httptest"
|
||||||
"net/url"
|
|
||||||
"strings"
|
|
||||||
"sync"
|
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
@@ -21,21 +16,11 @@ import (
|
|||||||
// pinned without standing up a real raft group.
|
// pinned without standing up a real raft group.
|
||||||
type stubCluster struct {
|
type stubCluster struct {
|
||||||
stats ClusterStats
|
stats ClusterStats
|
||||||
applyIndex uint64
|
|
||||||
applyErr error
|
|
||||||
applyCalled int
|
|
||||||
applyMu sync.Mutex
|
|
||||||
clusterIDOut string
|
clusterIDOut string
|
||||||
addrOut string
|
addrOut string
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *stubCluster) Stats() ClusterStats { return s.stats }
|
func (s *stubCluster) Stats() ClusterStats { return s.stats }
|
||||||
func (s *stubCluster) ApplyTestConfig() (uint64, error) {
|
|
||||||
s.applyMu.Lock()
|
|
||||||
defer s.applyMu.Unlock()
|
|
||||||
s.applyCalled++
|
|
||||||
return s.applyIndex, s.applyErr
|
|
||||||
}
|
|
||||||
func (s *stubCluster) ClusterID() string { return s.clusterIDOut }
|
func (s *stubCluster) ClusterID() string { return s.clusterIDOut }
|
||||||
func (s *stubCluster) LocalAddr() string { return s.addrOut }
|
func (s *stubCluster) LocalAddr() string { return s.addrOut }
|
||||||
|
|
||||||
@@ -166,151 +151,23 @@ func TestClusterStatus_RequiresSession(t *testing.T) {
|
|||||||
assert.Equal(t, "/web/login", resp.Header.Get("Location"))
|
assert.Equal(t, "/web/login", resp.Header.Get("Location"))
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestClusterApplyTestConfig_NotConfigured verifies the 404 path
|
// TestClusterApplyTestConfig_NotExposed verifies production requests cannot
|
||||||
// when WORKER_CLUSTER_DEBUG_APPLY is false (the production default)
|
// append a hardcoded config through the former debug endpoint.
|
||||||
// and no cluster is attached. The handler must refuse before it
|
func TestClusterApplyTestConfig_NotExposed(t *testing.T) {
|
||||||
// even checks the cluster because the debug flag is off.
|
t.Setenv("WORKER_CLUSTER_DEBUG_APPLY", "true")
|
||||||
func TestClusterApplyTestConfig_NotConfigured(t *testing.T) {
|
ts, _, c := withClusterServer(t, &stubCluster{})
|
||||||
srv := newTestServer(t, &stubRunner{id: "w-1"})
|
|
||||||
require.False(t, srv.cfg.DebugClusterApply, "default config must leave the debug apply flag off")
|
|
||||||
ts := newHTTPTestServer(t, srv)
|
|
||||||
c, _ := loginAsFirstRun(t, ts.URL, srv)
|
|
||||||
clearRequiresChange(t, srv)
|
|
||||||
|
|
||||||
resp, err := c.PostForm(ts.URL+"/web/api/cluster/apply-test-config", url.Values{})
|
resp, err := c.Post(ts.URL+"/web/api/cluster/apply-test-config", "", nil)
|
||||||
require.NoError(t, err)
|
|
||||||
defer resp.Body.Close() //nolint:errcheck
|
|
||||||
assert.Equal(t, http.StatusNotFound, resp.StatusCode,
|
|
||||||
"debug apply must be invisible (404) when WORKER_CLUSTER_DEBUG_APPLY is unset")
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestClusterApplyTestConfig_DebugOffReturns404 verifies that even
|
|
||||||
// with a cluster attached the apply endpoint stays 404 unless the
|
|
||||||
// debug flag is on. The flag, not cluster presence, gates the
|
|
||||||
// endpoint.
|
|
||||||
func TestClusterApplyTestConfig_DebugOffReturns404(t *testing.T) {
|
|
||||||
stub := &stubCluster{applyIndex: 42}
|
|
||||||
ts, _, c := withClusterServer(t, stub)
|
|
||||||
|
|
||||||
resp, err := c.Get(ts.URL + "/overview")
|
|
||||||
require.NoError(t, err)
|
|
||||||
bodyBytes, _ := io.ReadAll(resp.Body)
|
|
||||||
resp.Body.Close() //nolint:errcheck
|
|
||||||
csrf := extractCSRFToken(t, string(bodyBytes))
|
|
||||||
|
|
||||||
form := url.Values{}
|
|
||||||
form.Set("csrf_token", csrf)
|
|
||||||
req, _ := http.NewRequest(http.MethodPost, ts.URL+"/web/api/cluster/apply-test-config",
|
|
||||||
strings.NewReader(form.Encode()))
|
|
||||||
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
|
|
||||||
resp, err = c.Do(req)
|
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
defer resp.Body.Close() //nolint:errcheck
|
defer resp.Body.Close() //nolint:errcheck
|
||||||
assert.Equal(t, http.StatusNotFound, resp.StatusCode)
|
assert.Equal(t, http.StatusNotFound, resp.StatusCode)
|
||||||
assert.Equal(t, 0, stub.applyCalled,
|
|
||||||
"ApplyTestConfig must never be called when the debug flag is off")
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestClusterApplyTestConfig_HappyPath verifies that the apply-test-
|
|
||||||
// config endpoint returns the applied index when the cluster
|
|
||||||
// subsystem accepts the entry. CSRF is checked. The DebugClusterApply
|
|
||||||
// flag must be on for the endpoint to be reachable.
|
|
||||||
func TestClusterApplyTestConfig_HappyPath(t *testing.T) {
|
|
||||||
stub := &stubCluster{
|
|
||||||
stats: ClusterStats{
|
|
||||||
NodeID: "worker1", State: "Leader", Leader: "worker1",
|
|
||||||
Voters: []string{"worker1"},
|
|
||||||
},
|
|
||||||
applyIndex: 13,
|
|
||||||
clusterIDOut: "worker1",
|
|
||||||
addrOut: "127.0.0.1:17401",
|
|
||||||
}
|
|
||||||
ts, srv, c := withClusterServer(t, stub)
|
|
||||||
srv.cfg.DebugClusterApply = true
|
|
||||||
|
|
||||||
// Fetch CSRF token from any authenticated page.
|
|
||||||
resp, err := c.Get(ts.URL + "/overview")
|
|
||||||
require.NoError(t, err)
|
|
||||||
bodyBytes, _ := io.ReadAll(resp.Body)
|
|
||||||
resp.Body.Close() //nolint:errcheck
|
|
||||||
csrf := extractCSRFToken(t, string(bodyBytes))
|
|
||||||
|
|
||||||
form := url.Values{}
|
|
||||||
form.Set("csrf_token", csrf)
|
|
||||||
req, _ := http.NewRequest(http.MethodPost, ts.URL+"/web/api/cluster/apply-test-config",
|
|
||||||
strings.NewReader(form.Encode()))
|
|
||||||
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
|
|
||||||
resp, err = c.Do(req)
|
|
||||||
require.NoError(t, err)
|
|
||||||
defer resp.Body.Close() //nolint:errcheck
|
|
||||||
assert.Equal(t, http.StatusOK, resp.StatusCode)
|
|
||||||
bodyBytes, _ = io.ReadAll(resp.Body)
|
|
||||||
|
|
||||||
var got map[string]uint64
|
|
||||||
require.NoError(t, json.Unmarshal(bodyBytes, &got))
|
|
||||||
assert.EqualValues(t, 13, got["applied_index"])
|
|
||||||
|
|
||||||
assert.Equal(t, 1, stub.applyCalled)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestClusterApplyTestConfig_PropagatesError verifies that errors
|
|
||||||
// from the cluster subsystem surface as 502 Bad Gateway. Debug flag
|
|
||||||
// must be on.
|
|
||||||
func TestClusterApplyTestConfig_PropagatesError(t *testing.T) {
|
|
||||||
stub := &stubCluster{
|
|
||||||
applyErr: errStubApply,
|
|
||||||
clusterIDOut: "worker1",
|
|
||||||
addrOut: "127.0.0.1:17401",
|
|
||||||
}
|
|
||||||
ts, srv, c := withClusterServer(t, stub)
|
|
||||||
srv.cfg.DebugClusterApply = true
|
|
||||||
|
|
||||||
resp, err := c.Get(ts.URL + "/overview")
|
|
||||||
require.NoError(t, err)
|
|
||||||
bodyBytes, _ := io.ReadAll(resp.Body)
|
|
||||||
resp.Body.Close() //nolint:errcheck
|
|
||||||
csrf := extractCSRFToken(t, string(bodyBytes))
|
|
||||||
|
|
||||||
form := url.Values{}
|
|
||||||
form.Set("csrf_token", csrf)
|
|
||||||
req, _ := http.NewRequest(http.MethodPost, ts.URL+"/web/api/cluster/apply-test-config",
|
|
||||||
strings.NewReader(form.Encode()))
|
|
||||||
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
|
|
||||||
resp, err = c.Do(req)
|
|
||||||
require.NoError(t, err)
|
|
||||||
defer resp.Body.Close() //nolint:errcheck
|
|
||||||
assert.Equal(t, http.StatusBadGateway, resp.StatusCode)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestClusterApplyTestConfig_RequiresCSRF ensures the apply-test-
|
|
||||||
// config POST is refused without a CSRF token. Debug flag must be
|
|
||||||
// on for the endpoint to be reachable; without the flag it returns
|
|
||||||
// 404 (priority over CSRF check).
|
|
||||||
func TestClusterApplyTestConfig_RequiresCSRF(t *testing.T) {
|
|
||||||
stub := &stubCluster{applyIndex: 99}
|
|
||||||
ts, srv, c := withClusterServer(t, stub)
|
|
||||||
srv.cfg.DebugClusterApply = true
|
|
||||||
|
|
||||||
resp, err := c.PostForm(ts.URL+"/web/api/cluster/apply-test-config", url.Values{})
|
|
||||||
require.NoError(t, err)
|
|
||||||
defer resp.Body.Close() //nolint:errcheck
|
|
||||||
assert.Equal(t, http.StatusForbidden, resp.StatusCode,
|
|
||||||
"apply-test-config without CSRF must be 403")
|
|
||||||
assert.Equal(t, 0, stub.applyCalled, "ApplyTestConfig must not be called without CSRF")
|
|
||||||
}
|
|
||||||
|
|
||||||
// errStubApply is a sentinel error used by the apply-error test.
|
|
||||||
var errStubApply = errApply("worker not leader")
|
|
||||||
|
|
||||||
type errApply string
|
|
||||||
|
|
||||||
func (e errApply) Error() string { return string(e) }
|
|
||||||
|
|
||||||
// TestSetClusterDetaches verifies SetCluster(nil) returns the server
|
// TestSetClusterDetaches verifies SetCluster(nil) returns the server
|
||||||
// to the no-cluster-attached state (503 from the endpoints).
|
// to the no-cluster-attached state (503 from the endpoints).
|
||||||
func TestSetClusterDetaches(t *testing.T) {
|
func TestSetClusterDetaches(t *testing.T) {
|
||||||
srv := newTestServer(t, &stubRunner{id: "w-1"})
|
srv := newTestServer(t, &stubRunner{id: "w-1"})
|
||||||
stub := &stubCluster{applyIndex: 7}
|
stub := &stubCluster{}
|
||||||
srv.SetCluster(stub)
|
srv.SetCluster(stub)
|
||||||
require.NotNil(t, srv.Cluster())
|
require.NotNil(t, srv.Cluster())
|
||||||
|
|
||||||
@@ -326,11 +183,3 @@ func TestSetClusterDetaches(t *testing.T) {
|
|||||||
defer resp.Body.Close() //nolint:errcheck
|
defer resp.Body.Close() //nolint:errcheck
|
||||||
assert.Equal(t, http.StatusServiceUnavailable, resp.StatusCode)
|
assert.Equal(t, http.StatusServiceUnavailable, resp.StatusCode)
|
||||||
}
|
}
|
||||||
|
|
||||||
// _ = context.Background and time.Time keep the linter quiet about
|
|
||||||
// unused imports if the file shrinks.
|
|
||||||
var (
|
|
||||||
_ = context.Background
|
|
||||||
_ = time.Now
|
|
||||||
_ = url.Parse
|
|
||||||
)
|
|
||||||
|
|||||||
280
internal/webapp/handlers_compose.go
Обычный файл
280
internal/webapp/handlers_compose.go
Обычный файл
@@ -0,0 +1,280 @@
|
|||||||
|
package webapp
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"net/http"
|
||||||
|
"net/url"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"rocketgit.ru/rsmon/worker/internal/compose"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Compose management endpoints. The browser UI uses session+CSRF POSTs
|
||||||
|
// under /compose; scripted callers use HTTP basic auth under
|
||||||
|
// /web/api/compose (the /web/api/* prefix is the only one the auth
|
||||||
|
// middleware accepts basic credentials on). Read endpoints (list,
|
||||||
|
// detail, logs) are GET; lifecycle endpoints (up/down/stop/restart/pull)
|
||||||
|
// are POST. Every mutation is audited via the shared webapp audit log.
|
||||||
|
|
||||||
|
// handleComposeList renders the project list page.
|
||||||
|
func (s *Server) handleComposeList(w http.ResponseWriter, r *http.Request) {
|
||||||
|
writeNoStore(w)
|
||||||
|
sess, _ := sessionFromContext(r.Context())
|
||||||
|
data := composeListPageData{
|
||||||
|
basePageData: s.newBasePage(r, "Compose projects", sess),
|
||||||
|
Snapshot: s.compose.Snapshot(),
|
||||||
|
LastAt: s.compose.LastRefreshAt(),
|
||||||
|
Enabled: s.compose.Enabled(),
|
||||||
|
}
|
||||||
|
if err := s.templates.Execute(w, "compose.html", data); err != nil {
|
||||||
|
s.deps.Logger.Printf("render compose: %v", err)
|
||||||
|
http.Error(w, "template error", http.StatusInternalServerError)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// handleComposeDetail renders one project with its services, containers,
|
||||||
|
// mounts, and Traefik routes, plus the management action buttons.
|
||||||
|
func (s *Server) handleComposeDetail(w http.ResponseWriter, r *http.Request) {
|
||||||
|
writeNoStore(w)
|
||||||
|
sess, _ := sessionFromContext(r.Context())
|
||||||
|
snap := s.compose.Snapshot()
|
||||||
|
p := snap.FindProject(r.PathValue("project"))
|
||||||
|
if p == nil {
|
||||||
|
http.NotFound(w, r)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
data := composeDetailPageData{
|
||||||
|
basePageData: s.newBasePage(r, "Compose: "+p.Name, sess),
|
||||||
|
Project: p,
|
||||||
|
LastAt: s.compose.LastRefreshAt(),
|
||||||
|
ActionOK: r.URL.Query().Get("ok") == "1",
|
||||||
|
ActionMessage: r.URL.Query().Get("msg"),
|
||||||
|
}
|
||||||
|
if err := s.templates.Execute(w, "compose_detail.html", data); err != nil {
|
||||||
|
s.deps.Logger.Printf("render compose detail: %v", err)
|
||||||
|
http.Error(w, "template error", http.StatusInternalServerError)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// handleComposeLogsPage renders the recent log tail for a project in a
|
||||||
|
// <pre> block so an operator can read it from the browser without curl.
|
||||||
|
func (s *Server) handleComposeLogsPage(w http.ResponseWriter, r *http.Request) {
|
||||||
|
writeNoStore(w)
|
||||||
|
sess, _ := sessionFromContext(r.Context())
|
||||||
|
project := r.PathValue("project")
|
||||||
|
res, status := s.composeLogsResult(r.Context(), project)
|
||||||
|
if status != http.StatusOK {
|
||||||
|
http.Error(w, res.Output, status)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
data := composeLogsPageData{
|
||||||
|
basePageData: s.newBasePage(r, "Compose logs: "+project, sess),
|
||||||
|
Project: project,
|
||||||
|
Output: res.Output,
|
||||||
|
}
|
||||||
|
if err := s.templates.Execute(w, "compose_logs.html", data); err != nil {
|
||||||
|
s.deps.Logger.Printf("render compose logs: %v", err)
|
||||||
|
http.Error(w, "template error", http.StatusInternalServerError)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// handleComposeProjectAction runs a lifecycle action against a project
|
||||||
|
// from the HTML form (session+CSRF) or the API (basic auth). The
|
||||||
|
// response is a redirect for the browser path and JSON for the API path.
|
||||||
|
func (s *Server) handleComposeProjectAction(w http.ResponseWriter, r *http.Request) {
|
||||||
|
if !s.compose.Enabled() {
|
||||||
|
http.Error(w, "compose management is disabled", http.StatusServiceUnavailable)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if !validProjectPostAction(r.PathValue("action")) {
|
||||||
|
http.NotFound(w, r)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if !s.composeMutationAuthorized(w, r) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
project := r.PathValue("project")
|
||||||
|
action := r.PathValue("action")
|
||||||
|
refs, ok := s.compose.Refs(project)
|
||||||
|
if !ok {
|
||||||
|
http.Error(w, "compose project working directory is unknown", http.StatusNotFound)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
res, _ := compose.ManageProject(r.Context(), refs, compose.Action(action))
|
||||||
|
s.auditCompose(r, action, project, "")
|
||||||
|
s.respondComposeAction(w, r, project, res)
|
||||||
|
}
|
||||||
|
|
||||||
|
// handleComposeServiceAction runs a lifecycle action against a single
|
||||||
|
// service inside a project.
|
||||||
|
func (s *Server) handleComposeServiceAction(w http.ResponseWriter, r *http.Request) {
|
||||||
|
if !s.compose.Enabled() {
|
||||||
|
http.Error(w, "compose management is disabled", http.StatusServiceUnavailable)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if _, ok := compose.ValidServiceAction(r.PathValue("action")); !ok {
|
||||||
|
http.NotFound(w, r)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if !s.composeMutationAuthorized(w, r) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
project := r.PathValue("project")
|
||||||
|
service := r.PathValue("service")
|
||||||
|
action := r.PathValue("action")
|
||||||
|
refs, ok := s.compose.Refs(project)
|
||||||
|
if !ok {
|
||||||
|
http.Error(w, "compose project working directory is unknown", http.StatusNotFound)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
res, _ := compose.ManageService(r.Context(), refs, service, compose.Action(action))
|
||||||
|
s.auditCompose(r, action, project, service)
|
||||||
|
s.respondComposeAction(w, r, project, res)
|
||||||
|
}
|
||||||
|
|
||||||
|
// handleComposeAPIList returns the full discovery snapshot as JSON. The
|
||||||
|
// shape matches the /compose HTML page (a Snapshot: projects + errors).
|
||||||
|
func (s *Server) handleComposeAPIList(w http.ResponseWriter, r *http.Request) {
|
||||||
|
writeComposeJSON(w, s.compose.Snapshot())
|
||||||
|
}
|
||||||
|
|
||||||
|
// handleComposeAPIDetail returns one project summary as JSON.
|
||||||
|
func (s *Server) handleComposeAPIDetail(w http.ResponseWriter, r *http.Request) {
|
||||||
|
p := s.compose.Snapshot().FindProject(r.PathValue("project"))
|
||||||
|
if p == nil {
|
||||||
|
http.Error(w, "compose project not found", http.StatusNotFound)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
writeComposeJSON(w, p)
|
||||||
|
}
|
||||||
|
|
||||||
|
// handleComposeLogsAPI returns the recent log tail for a project as JSON.
|
||||||
|
func (s *Server) handleComposeLogsAPI(w http.ResponseWriter, r *http.Request) {
|
||||||
|
res, status := s.composeLogsResult(r.Context(), r.PathValue("project"))
|
||||||
|
if status != http.StatusOK {
|
||||||
|
w.Header().Set("Content-Type", "application/json; charset=utf-8")
|
||||||
|
w.WriteHeader(status)
|
||||||
|
_ = json.NewEncoder(w).Encode(res)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
writeComposeJSON(w, res)
|
||||||
|
}
|
||||||
|
|
||||||
|
// composeLogsResult resolves the log tail for a project and the HTTP
|
||||||
|
// status the caller should write. A 404 means the project (or its
|
||||||
|
// working directory) is unknown; 502 means Docker refused the call.
|
||||||
|
func (s *Server) composeLogsResult(ctx context.Context, project string) (compose.ManagementResult, int) {
|
||||||
|
refs, ok := s.compose.Refs(project)
|
||||||
|
if !ok {
|
||||||
|
return compose.ManagementResult{OK: false, Action: "logs", Output: "compose project working directory is unknown"}, http.StatusNotFound
|
||||||
|
}
|
||||||
|
res, err := compose.Logs(ctx, refs, 300)
|
||||||
|
if err != nil && res.Output == "" {
|
||||||
|
return res, http.StatusBadGateway
|
||||||
|
}
|
||||||
|
return res, http.StatusOK
|
||||||
|
}
|
||||||
|
|
||||||
|
// composeMutationAuthorized enforces CSRF for browser (session) callers
|
||||||
|
// while letting basic-auth API callers through without a token (the
|
||||||
|
// middleware already validated their credentials). Returns true when the
|
||||||
|
// request may proceed; writes a 403 and returns false otherwise.
|
||||||
|
func (s *Server) composeMutationAuthorized(w http.ResponseWriter, r *http.Request) bool {
|
||||||
|
sess, hasSess := sessionFromContext(r.Context())
|
||||||
|
if !hasSess {
|
||||||
|
return true // basic-auth API path; middleware authenticated it
|
||||||
|
}
|
||||||
|
if !s.requireCSRF(sess, r) {
|
||||||
|
http.Error(w, "csrf token required", http.StatusForbidden)
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
// validProjectPostAction reports whether name is a POST-able project
|
||||||
|
// action. It reuses compose.ValidProjectAction but excludes "logs",
|
||||||
|
// which is served by a dedicated GET route.
|
||||||
|
func validProjectPostAction(name string) bool {
|
||||||
|
if name == string(compose.ActionLogs) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
_, ok := compose.ValidProjectAction(name)
|
||||||
|
return ok
|
||||||
|
}
|
||||||
|
|
||||||
|
// respondComposeAction sends the action result to the right consumer:
|
||||||
|
// JSON for /web/api/* (scripting), a redirect with a flash banner for
|
||||||
|
// the browser path. The action's combined output is URL-encoded into the
|
||||||
|
// banner so the operator sees what Compose printed without a separate
|
||||||
|
// request; very large output is truncated to keep the URL bounded.
|
||||||
|
func (s *Server) respondComposeAction(w http.ResponseWriter, r *http.Request, project string, res compose.ManagementResult) {
|
||||||
|
if strings.HasPrefix(r.URL.Path, "/web/api/") {
|
||||||
|
writeComposeJSON(w, res)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
msg := strings.TrimSpace(res.Output)
|
||||||
|
if len(msg) > 500 {
|
||||||
|
msg = msg[:500] + "…"
|
||||||
|
}
|
||||||
|
if !res.OK && msg == "" {
|
||||||
|
msg = res.Action + " failed"
|
||||||
|
}
|
||||||
|
loc := fmt.Sprintf("/compose/%s?ok=%t&msg=%s", project, res.OK, url.QueryEscape(msg))
|
||||||
|
http.Redirect(w, r, loc, http.StatusSeeOther)
|
||||||
|
}
|
||||||
|
|
||||||
|
// writeComposeJSON serializes a value as no-store JSON. Used by every
|
||||||
|
// /web/api/compose endpoint so scripted callers get one consistent shape.
|
||||||
|
func writeComposeJSON(w http.ResponseWriter, v interface{}) {
|
||||||
|
writeNoStore(w)
|
||||||
|
w.Header().Set("Content-Type", "application/json; charset=utf-8")
|
||||||
|
_ = json.NewEncoder(w).Encode(v)
|
||||||
|
}
|
||||||
|
|
||||||
|
// auditCompose records a management action in the worker audit log. The
|
||||||
|
// target encodes project and service so the audit trail reconstructs the
|
||||||
|
// exact scope (e.g. "compose:rsmon/web"); failures are still audited so
|
||||||
|
// an operator can trace a stopped service back to the action.
|
||||||
|
func (s *Server) auditCompose(r *http.Request, action, project, service string) {
|
||||||
|
target := "compose:" + project
|
||||||
|
if service != "" {
|
||||||
|
target += "/" + service
|
||||||
|
}
|
||||||
|
authMode := auditAuthModeBasic
|
||||||
|
if _, hasSess := sessionFromContext(r.Context()); hasSess {
|
||||||
|
authMode = auditAuthModeLocal
|
||||||
|
}
|
||||||
|
_ = s.store.WriteAudit(r.Context(), &AuditEntry{
|
||||||
|
Actor: auditActorLocal,
|
||||||
|
Role: auditRoleAdmin,
|
||||||
|
AuthMode: authMode,
|
||||||
|
IP: clientIP(r),
|
||||||
|
UA: r.UserAgent(),
|
||||||
|
Action: "compose_" + action,
|
||||||
|
Target: target,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
type composeListPageData struct {
|
||||||
|
basePageData
|
||||||
|
Snapshot compose.Snapshot
|
||||||
|
LastAt time.Time
|
||||||
|
Enabled bool
|
||||||
|
}
|
||||||
|
|
||||||
|
type composeDetailPageData struct {
|
||||||
|
basePageData
|
||||||
|
Project *compose.ProjectSummary
|
||||||
|
LastAt time.Time
|
||||||
|
ActionOK bool
|
||||||
|
ActionMessage string
|
||||||
|
}
|
||||||
|
|
||||||
|
type composeLogsPageData struct {
|
||||||
|
basePageData
|
||||||
|
Project string
|
||||||
|
Output string
|
||||||
|
}
|
||||||
417
internal/webapp/handlers_compose_test.go
Обычный файл
417
internal/webapp/handlers_compose_test.go
Обычный файл
@@ -0,0 +1,417 @@
|
|||||||
|
package webapp
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"net/http"
|
||||||
|
"net/url"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
|
||||||
|
"rocketgit.ru/rsmon/worker/internal/compose"
|
||||||
|
)
|
||||||
|
|
||||||
|
// newComposeTestServer builds a session-auth server with the Compose
|
||||||
|
// subsystem enabled. The default newTestServer leaves ComposeEnabled
|
||||||
|
// false (zero-value Config), so the handlers would short-circuit to the
|
||||||
|
// disabled banner; flipping the refresher flag on exercises the real
|
||||||
|
// code paths without starting the background loop.
|
||||||
|
func newComposeTestServer(t *testing.T, runner WorkerView) *Server {
|
||||||
|
t.Helper()
|
||||||
|
srv := newTestServer(t, runner)
|
||||||
|
srv.compose.enabled = true
|
||||||
|
return srv
|
||||||
|
}
|
||||||
|
|
||||||
|
// newComposeTestServerBasicAuth is the basic-auth variant used by the
|
||||||
|
// /web/api/compose/* tests: those routes accept HTTP basic credentials
|
||||||
|
// in lieu of a session cookie.
|
||||||
|
func newComposeTestServerBasicAuth(t *testing.T, runner WorkerView, login, password string) *Server {
|
||||||
|
t.Helper()
|
||||||
|
srv := newTestServerWithBasicAuth(t, runner, login, password)
|
||||||
|
srv.compose.enabled = true
|
||||||
|
return srv
|
||||||
|
}
|
||||||
|
|
||||||
|
// injectComposeSnapshot seeds the refresher with a single "rsmon"
|
||||||
|
// project so the list/detail/logs handlers have something to render
|
||||||
|
// without exec'ing Docker. The project's WorkingDir is a real temp dir
|
||||||
|
// so management operations (which chdir into it) succeed under a stub
|
||||||
|
// Docker binary.
|
||||||
|
func injectComposeSnapshot(t *testing.T, srv *Server, workDir string) {
|
||||||
|
t.Helper()
|
||||||
|
res := &compose.DiscoveryResult{
|
||||||
|
Projects: map[string]compose.Project{
|
||||||
|
"rsmon": {
|
||||||
|
Name: "rsmon",
|
||||||
|
Status: "running(1)",
|
||||||
|
WorkingDir: workDir,
|
||||||
|
ConfigFiles: filepath.Join(workDir, "docker-compose.yml"),
|
||||||
|
Services: map[string]compose.Service{
|
||||||
|
"web": {Name: "web", Containers: []compose.Container{{
|
||||||
|
ID: "c1",
|
||||||
|
Name: "rsmon-web-1",
|
||||||
|
Image: "nginx:latest",
|
||||||
|
State: "running",
|
||||||
|
Status: "Up 5 minutes",
|
||||||
|
Health: "healthy",
|
||||||
|
PID: 4242,
|
||||||
|
Ports: []compose.PortBinding{{HostIP: "0.0.0.0", HostPort: "8080"}},
|
||||||
|
Mounts: []compose.Mount{{Source: filepath.Join(workDir, "data"), Destination: "/data", Type: "bind"}},
|
||||||
|
Labels: map[string]string{
|
||||||
|
compose.LabelComposeProject: "rsmon",
|
||||||
|
compose.LabelComposeService: "web",
|
||||||
|
},
|
||||||
|
}}},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
snap := compose.Summarize(res)
|
||||||
|
srv.compose.mu.Lock()
|
||||||
|
srv.compose.snap = snap
|
||||||
|
srv.compose.lastAt = time.Now().UTC()
|
||||||
|
srv.compose.mu.Unlock()
|
||||||
|
}
|
||||||
|
|
||||||
|
// stubDockerOK points the compose package at an executable stub that
|
||||||
|
// echoes its arguments and exits 0, so management/logs handlers run the
|
||||||
|
// full exec path without a real Docker daemon. Restored on cleanup.
|
||||||
|
func stubDockerOK(t *testing.T) {
|
||||||
|
t.Helper()
|
||||||
|
dir := t.TempDir()
|
||||||
|
p := filepath.Join(dir, "stub-docker")
|
||||||
|
script := "#!/bin/sh\necho \"stub: $*\"\nexit 0\n"
|
||||||
|
require.NoError(t, os.WriteFile(p, []byte(script), 0o755))
|
||||||
|
compose.SetDockerBin(p)
|
||||||
|
t.Cleanup(func() { compose.SetDockerBin("") })
|
||||||
|
}
|
||||||
|
|
||||||
|
// stubDockerFail points at a stub that exits non-zero with a stderr
|
||||||
|
// message, so the !OK management path is observable.
|
||||||
|
func stubDockerFail(t *testing.T) {
|
||||||
|
t.Helper()
|
||||||
|
dir := t.TempDir()
|
||||||
|
p := filepath.Join(dir, "stub-docker")
|
||||||
|
script := "#!/bin/sh\necho \"compose boom\" >&2\nexit 1\n"
|
||||||
|
require.NoError(t, os.WriteFile(p, []byte(script), 0o755))
|
||||||
|
compose.SetDockerBin(p)
|
||||||
|
t.Cleanup(func() { compose.SetDockerBin("") })
|
||||||
|
}
|
||||||
|
|
||||||
|
// auditHasAction reports whether the audit log contains a row matching
|
||||||
|
// action and a target substring (e.g. "compose_restart" / "rsmon").
|
||||||
|
func auditHasAction(t *testing.T, srv *Server, action, targetSub string) bool {
|
||||||
|
t.Helper()
|
||||||
|
rows, err := srv.store.RecentAudit(context.Background(), 10)
|
||||||
|
require.NoError(t, err)
|
||||||
|
for _, r := range rows {
|
||||||
|
if r.Action == action && strings.Contains(r.Target, targetSub) {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestComposeList_RendersProjects(t *testing.T) {
|
||||||
|
srv := newComposeTestServer(t, &stubRunner{id: "w-1"})
|
||||||
|
injectComposeSnapshot(t, srv, t.TempDir())
|
||||||
|
ts := newHTTPTestServer(t, srv)
|
||||||
|
|
||||||
|
c, _ := loginAsFirstRun(t, ts.URL, srv)
|
||||||
|
resp, err := c.Get(ts.URL + "/compose")
|
||||||
|
require.NoError(t, err)
|
||||||
|
body := mustBody(t, resp)
|
||||||
|
assert.Contains(t, body, "Docker Compose projects")
|
||||||
|
assert.Contains(t, body, "rsmon")
|
||||||
|
assert.Contains(t, body, "details") // link to the detail page
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestComposeList_DisabledBanner(t *testing.T) {
|
||||||
|
// Default server leaves Compose disabled.
|
||||||
|
srv := newTestServer(t, &stubRunner{id: "w-1"})
|
||||||
|
ts := newHTTPTestServer(t, srv)
|
||||||
|
|
||||||
|
c, _ := loginAsFirstRun(t, ts.URL, srv)
|
||||||
|
resp, err := c.Get(ts.URL + "/compose")
|
||||||
|
require.NoError(t, err)
|
||||||
|
body := mustBody(t, resp)
|
||||||
|
assert.Contains(t, body, "Compose management is disabled")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestComposeDetail_FoundAndNotFound(t *testing.T) {
|
||||||
|
srv := newComposeTestServer(t, &stubRunner{id: "w-1"})
|
||||||
|
injectComposeSnapshot(t, srv, t.TempDir())
|
||||||
|
ts := newHTTPTestServer(t, srv)
|
||||||
|
|
||||||
|
c, _ := loginAsFirstRun(t, ts.URL, srv)
|
||||||
|
|
||||||
|
resp, err := c.Get(ts.URL + "/compose/rsmon")
|
||||||
|
require.NoError(t, err)
|
||||||
|
body := mustBody(t, resp)
|
||||||
|
assert.Contains(t, body, "Project actions")
|
||||||
|
assert.Contains(t, body, "rsmon-web-1")
|
||||||
|
|
||||||
|
resp2, err := c.Get(ts.URL + "/compose/missing")
|
||||||
|
require.NoError(t, err)
|
||||||
|
defer resp2.Body.Close() //nolint:errcheck
|
||||||
|
assert.Equal(t, http.StatusNotFound, resp2.StatusCode)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestComposeAPIList_JSON(t *testing.T) {
|
||||||
|
srv := newComposeTestServerBasicAuth(t, &stubRunner{id: "w-1"}, "alice", "s3cret")
|
||||||
|
injectComposeSnapshot(t, srv, t.TempDir())
|
||||||
|
ts := newHTTPTestServer(t, srv)
|
||||||
|
|
||||||
|
req, _ := http.NewRequest(http.MethodGet, ts.URL+"/web/api/compose", nil)
|
||||||
|
req.SetBasicAuth("alice", "s3cret")
|
||||||
|
resp, err := http.DefaultClient.Do(req)
|
||||||
|
require.NoError(t, err)
|
||||||
|
defer resp.Body.Close() //nolint:errcheck
|
||||||
|
require.Equal(t, http.StatusOK, resp.StatusCode)
|
||||||
|
|
||||||
|
var snap compose.Snapshot
|
||||||
|
require.NoError(t, json.NewDecoder(resp.Body).Decode(&snap))
|
||||||
|
require.Len(t, snap.Projects, 1)
|
||||||
|
assert.Equal(t, "rsmon", snap.Projects[0].Name)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestComposeAPIDetail_JSONAndNotFound(t *testing.T) {
|
||||||
|
srv := newComposeTestServerBasicAuth(t, &stubRunner{id: "w-1"}, "alice", "s3cret")
|
||||||
|
injectComposeSnapshot(t, srv, t.TempDir())
|
||||||
|
ts := newHTTPTestServer(t, srv)
|
||||||
|
|
||||||
|
req, _ := http.NewRequest(http.MethodGet, ts.URL+"/web/api/compose/rsmon", nil)
|
||||||
|
req.SetBasicAuth("alice", "s3cret")
|
||||||
|
resp, err := http.DefaultClient.Do(req)
|
||||||
|
require.NoError(t, err)
|
||||||
|
defer resp.Body.Close() //nolint:errcheck
|
||||||
|
require.Equal(t, http.StatusOK, resp.StatusCode)
|
||||||
|
var p compose.ProjectSummary
|
||||||
|
require.NoError(t, json.NewDecoder(resp.Body).Decode(&p))
|
||||||
|
assert.Equal(t, "rsmon", p.Name)
|
||||||
|
|
||||||
|
req2, _ := http.NewRequest(http.MethodGet, ts.URL+"/web/api/compose/missing", nil)
|
||||||
|
req2.SetBasicAuth("alice", "s3cret")
|
||||||
|
resp2, err := http.DefaultClient.Do(req2)
|
||||||
|
require.NoError(t, err)
|
||||||
|
defer resp2.Body.Close() //nolint:errcheck
|
||||||
|
assert.Equal(t, http.StatusNotFound, resp2.StatusCode)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestComposeProjectAction_RequiresCSRF(t *testing.T) {
|
||||||
|
srv := newComposeTestServer(t, &stubRunner{id: "w-1"})
|
||||||
|
injectComposeSnapshot(t, srv, t.TempDir())
|
||||||
|
ts := newHTTPTestServer(t, srv)
|
||||||
|
|
||||||
|
c, _ := loginAsFirstRun(t, ts.URL, srv)
|
||||||
|
form := url.Values{}
|
||||||
|
form.Set("csrf_token", "") // missing token
|
||||||
|
req, _ := http.NewRequest(http.MethodPost, ts.URL+"/compose/rsmon/restart", strings.NewReader(form.Encode()))
|
||||||
|
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
|
||||||
|
resp, err := c.Do(req)
|
||||||
|
require.NoError(t, err)
|
||||||
|
defer resp.Body.Close() //nolint:errcheck
|
||||||
|
assert.Equal(t, http.StatusForbidden, resp.StatusCode, "POST without CSRF must be 403")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestComposeProjectAction_HTMLRedirectAndAudit(t *testing.T) {
|
||||||
|
srv := newComposeTestServer(t, &stubRunner{id: "w-1"})
|
||||||
|
injectComposeSnapshot(t, srv, t.TempDir())
|
||||||
|
stubDockerOK(t)
|
||||||
|
ts := newHTTPTestServer(t, srv)
|
||||||
|
|
||||||
|
c, _ := loginAsFirstRun(t, ts.URL, srv)
|
||||||
|
|
||||||
|
// Pull the CSRF token out of the detail page's action forms.
|
||||||
|
detail, err := c.Get(ts.URL + "/compose/rsmon")
|
||||||
|
require.NoError(t, err)
|
||||||
|
csrf := extractCSRFToken(t, mustBody(t, detail))
|
||||||
|
|
||||||
|
form := url.Values{}
|
||||||
|
form.Set("csrf_token", csrf)
|
||||||
|
req, _ := http.NewRequest(http.MethodPost, ts.URL+"/compose/rsmon/restart", strings.NewReader(form.Encode()))
|
||||||
|
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
|
||||||
|
resp, err := c.Do(req)
|
||||||
|
require.NoError(t, err)
|
||||||
|
defer resp.Body.Close() //nolint:errcheck
|
||||||
|
assert.Equal(t, http.StatusSeeOther, resp.StatusCode)
|
||||||
|
loc := resp.Header.Get("Location")
|
||||||
|
assert.True(t, strings.HasPrefix(loc, "/compose/rsmon?ok=true"), "redirect location=%q", loc)
|
||||||
|
|
||||||
|
assert.True(t, auditHasAction(t, srv, "compose_restart", "compose:rsmon"),
|
||||||
|
"audit row for project restart expected")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestComposeProjectAction_APIBasicAuthAndAudit(t *testing.T) {
|
||||||
|
srv := newComposeTestServerBasicAuth(t, &stubRunner{id: "w-1"}, "alice", "s3cret")
|
||||||
|
injectComposeSnapshot(t, srv, t.TempDir())
|
||||||
|
stubDockerOK(t)
|
||||||
|
ts := newHTTPTestServer(t, srv)
|
||||||
|
|
||||||
|
req, _ := http.NewRequest(http.MethodPost, ts.URL+"/web/api/compose/rsmon/restart", nil)
|
||||||
|
req.SetBasicAuth("alice", "s3cret")
|
||||||
|
resp, err := http.DefaultClient.Do(req)
|
||||||
|
require.NoError(t, err)
|
||||||
|
defer resp.Body.Close() //nolint:errcheck
|
||||||
|
require.Equal(t, http.StatusOK, resp.StatusCode)
|
||||||
|
|
||||||
|
var res compose.ManagementResult
|
||||||
|
require.NoError(t, json.NewDecoder(resp.Body).Decode(&res))
|
||||||
|
assert.True(t, res.OK)
|
||||||
|
assert.Contains(t, res.Output, "stub:")
|
||||||
|
|
||||||
|
assert.True(t, auditHasAction(t, srv, "compose_restart", "compose:rsmon"),
|
||||||
|
"audit row for API restart expected")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestComposeProjectAction_FailureReportsNotOK(t *testing.T) {
|
||||||
|
srv := newComposeTestServerBasicAuth(t, &stubRunner{id: "w-1"}, "alice", "s3cret")
|
||||||
|
injectComposeSnapshot(t, srv, t.TempDir())
|
||||||
|
stubDockerFail(t)
|
||||||
|
ts := newHTTPTestServer(t, srv)
|
||||||
|
|
||||||
|
req, _ := http.NewRequest(http.MethodPost, ts.URL+"/web/api/compose/rsmon/stop", nil)
|
||||||
|
req.SetBasicAuth("alice", "s3cret")
|
||||||
|
resp, err := http.DefaultClient.Do(req)
|
||||||
|
require.NoError(t, err)
|
||||||
|
defer resp.Body.Close() //nolint:errcheck
|
||||||
|
require.Equal(t, http.StatusOK, resp.StatusCode)
|
||||||
|
|
||||||
|
var res compose.ManagementResult
|
||||||
|
require.NoError(t, json.NewDecoder(resp.Body).Decode(&res))
|
||||||
|
assert.False(t, res.OK)
|
||||||
|
assert.Contains(t, res.Output, "compose boom")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestComposeProjectAction_UnknownActionRejected(t *testing.T) {
|
||||||
|
srv := newComposeTestServer(t, &stubRunner{id: "w-1"})
|
||||||
|
injectComposeSnapshot(t, srv, t.TempDir())
|
||||||
|
ts := newHTTPTestServer(t, srv)
|
||||||
|
|
||||||
|
c, _ := loginAsFirstRun(t, ts.URL, srv)
|
||||||
|
for _, action := range []string{"bogus", "logs"} { // logs is GET-only
|
||||||
|
req, _ := http.NewRequest(http.MethodPost, ts.URL+"/compose/rsmon/"+action, strings.NewReader(""))
|
||||||
|
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
|
||||||
|
resp, err := c.Do(req)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, http.StatusNotFound, resp.StatusCode, "action %q should 404", action)
|
||||||
|
resp.Body.Close() //nolint:errcheck
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestComposeProjectAction_DisabledReturns503(t *testing.T) {
|
||||||
|
srv := newTestServer(t, &stubRunner{id: "w-1"}) // compose disabled
|
||||||
|
injectComposeSnapshot(t, srv, t.TempDir())
|
||||||
|
ts := newHTTPTestServer(t, srv)
|
||||||
|
|
||||||
|
c, _ := loginAsFirstRun(t, ts.URL, srv)
|
||||||
|
req, _ := http.NewRequest(http.MethodPost, ts.URL+"/compose/rsmon/restart", strings.NewReader(""))
|
||||||
|
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
|
||||||
|
resp, err := c.Do(req)
|
||||||
|
require.NoError(t, err)
|
||||||
|
defer resp.Body.Close() //nolint:errcheck
|
||||||
|
assert.Equal(t, http.StatusServiceUnavailable, resp.StatusCode)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestComposeServiceAction_HTMLRedirectAndAudit(t *testing.T) {
|
||||||
|
srv := newComposeTestServer(t, &stubRunner{id: "w-1"})
|
||||||
|
injectComposeSnapshot(t, srv, t.TempDir())
|
||||||
|
stubDockerOK(t)
|
||||||
|
ts := newHTTPTestServer(t, srv)
|
||||||
|
|
||||||
|
c, _ := loginAsFirstRun(t, ts.URL, srv)
|
||||||
|
detail, err := c.Get(ts.URL + "/compose/rsmon")
|
||||||
|
require.NoError(t, err)
|
||||||
|
csrf := extractCSRFToken(t, mustBody(t, detail))
|
||||||
|
|
||||||
|
form := url.Values{}
|
||||||
|
form.Set("csrf_token", csrf)
|
||||||
|
req, _ := http.NewRequest(http.MethodPost, ts.URL+"/compose/rsmon/service/web/restart", strings.NewReader(form.Encode()))
|
||||||
|
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
|
||||||
|
resp, err := c.Do(req)
|
||||||
|
require.NoError(t, err)
|
||||||
|
defer resp.Body.Close() //nolint:errcheck
|
||||||
|
assert.Equal(t, http.StatusSeeOther, resp.StatusCode)
|
||||||
|
assert.True(t, strings.HasPrefix(resp.Header.Get("Location"), "/compose/rsmon"))
|
||||||
|
assert.True(t, auditHasAction(t, srv, "compose_restart", "compose:rsmon/web"),
|
||||||
|
"audit row for service restart expected")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestComposeServiceAction_UnknownActionRejected(t *testing.T) {
|
||||||
|
srv := newComposeTestServer(t, &stubRunner{id: "w-1"})
|
||||||
|
injectComposeSnapshot(t, srv, t.TempDir())
|
||||||
|
ts := newHTTPTestServer(t, srv)
|
||||||
|
|
||||||
|
c, _ := loginAsFirstRun(t, ts.URL, srv)
|
||||||
|
req, _ := http.NewRequest(http.MethodPost, ts.URL+"/compose/rsmon/service/web/bogus", strings.NewReader(""))
|
||||||
|
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
|
||||||
|
resp, err := c.Do(req)
|
||||||
|
require.NoError(t, err)
|
||||||
|
defer resp.Body.Close() //nolint:errcheck
|
||||||
|
assert.Equal(t, http.StatusNotFound, resp.StatusCode)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestComposeLogs_HTMLPage(t *testing.T) {
|
||||||
|
srv := newComposeTestServer(t, &stubRunner{id: "w-1"})
|
||||||
|
injectComposeSnapshot(t, srv, t.TempDir())
|
||||||
|
stubDockerOK(t)
|
||||||
|
ts := newHTTPTestServer(t, srv)
|
||||||
|
|
||||||
|
c, _ := loginAsFirstRun(t, ts.URL, srv)
|
||||||
|
resp, err := c.Get(ts.URL + "/compose/rsmon/logs")
|
||||||
|
require.NoError(t, err)
|
||||||
|
body := mustBody(t, resp)
|
||||||
|
assert.Contains(t, body, "Compose logs: rsmon")
|
||||||
|
assert.Contains(t, body, "stub:")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestComposeLogs_API_JSONAndNotFound(t *testing.T) {
|
||||||
|
srv := newComposeTestServerBasicAuth(t, &stubRunner{id: "w-1"}, "alice", "s3cret")
|
||||||
|
injectComposeSnapshot(t, srv, t.TempDir())
|
||||||
|
stubDockerOK(t)
|
||||||
|
ts := newHTTPTestServer(t, srv)
|
||||||
|
|
||||||
|
req, _ := http.NewRequest(http.MethodGet, ts.URL+"/web/api/compose/rsmon/logs", nil)
|
||||||
|
req.SetBasicAuth("alice", "s3cret")
|
||||||
|
resp, err := http.DefaultClient.Do(req)
|
||||||
|
require.NoError(t, err)
|
||||||
|
defer resp.Body.Close() //nolint:errcheck
|
||||||
|
require.Equal(t, http.StatusOK, resp.StatusCode)
|
||||||
|
var res compose.ManagementResult
|
||||||
|
require.NoError(t, json.NewDecoder(resp.Body).Decode(&res))
|
||||||
|
assert.Contains(t, res.Output, "stub:")
|
||||||
|
|
||||||
|
req2, _ := http.NewRequest(http.MethodGet, ts.URL+"/web/api/compose/missing/logs", nil)
|
||||||
|
req2.SetBasicAuth("alice", "s3cret")
|
||||||
|
resp2, err := http.DefaultClient.Do(req2)
|
||||||
|
require.NoError(t, err)
|
||||||
|
defer resp2.Body.Close() //nolint:errcheck
|
||||||
|
assert.Equal(t, http.StatusNotFound, resp2.StatusCode)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestValidProjectPostAction(t *testing.T) {
|
||||||
|
for _, a := range []string{"up", "down", "stop", "restart", "pull"} {
|
||||||
|
assert.True(t, validProjectPostAction(a), "%q should be POST-able", a)
|
||||||
|
}
|
||||||
|
assert.False(t, validProjectPostAction("logs"), "logs is GET-only")
|
||||||
|
assert.False(t, validProjectPostAction("bogus"), "bogus is invalid")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestComposeAPIList_RequiresAuth(t *testing.T) {
|
||||||
|
srv := newComposeTestServerBasicAuth(t, &stubRunner{id: "w-1"}, "alice", "s3cret")
|
||||||
|
injectComposeSnapshot(t, srv, t.TempDir())
|
||||||
|
ts := newHTTPTestServer(t, srv)
|
||||||
|
|
||||||
|
// No credentials: the basic-auth fast path returns 401.
|
||||||
|
resp, err := http.Get(ts.URL + "/web/api/compose")
|
||||||
|
require.NoError(t, err)
|
||||||
|
defer resp.Body.Close() //nolint:errcheck
|
||||||
|
assert.Equal(t, http.StatusUnauthorized, resp.StatusCode)
|
||||||
|
}
|
||||||
@@ -224,7 +224,10 @@ func TestChecksPage_RunNowDisabledButton(t *testing.T) {
|
|||||||
// TestNotificationsPage_ResendDisabledButton mirrors the checks
|
// TestNotificationsPage_ResendDisabledButton mirrors the checks
|
||||||
// page test for the resend button on /notifications.
|
// page test for the resend button on /notifications.
|
||||||
func TestNotificationsPage_ResendDisabledButton(t *testing.T) {
|
func TestNotificationsPage_ResendDisabledButton(t *testing.T) {
|
||||||
srv := newTestServer(t, &stubRunner{id: "w-1"})
|
srv := newTestServer(t, &stubRunner{id: "w-1", notifs: []NotificationRow{
|
||||||
|
{Kind: "email", Channel: "smtp", Subject: "selfcheck", Body: "main API down", OK: true, At: time.Now()},
|
||||||
|
{JobID: "delegated-job", Method: "sms", Status: "permanent", DurationMs: 12, At: time.Now()},
|
||||||
|
}})
|
||||||
ts := newHTTPTestServer(t, srv)
|
ts := newHTTPTestServer(t, srv)
|
||||||
c, _ := loginAsFirstRun(t, ts.URL, srv)
|
c, _ := loginAsFirstRun(t, ts.URL, srv)
|
||||||
clearRequiresChange(t, srv)
|
clearRequiresChange(t, srv)
|
||||||
@@ -239,6 +242,11 @@ func TestNotificationsPage_ResendDisabledButton(t *testing.T) {
|
|||||||
assert.Contains(t, page, "Resend")
|
assert.Contains(t, page, "Resend")
|
||||||
assert.Contains(t, page, "disabled")
|
assert.Contains(t, page, "disabled")
|
||||||
assert.Contains(t, page, "worker-notifier-mvp")
|
assert.Contains(t, page, "worker-notifier-mvp")
|
||||||
|
assert.Contains(t, page, "selfcheck")
|
||||||
|
assert.Contains(t, page, "delegated-job")
|
||||||
|
assert.Contains(t, page, "sms")
|
||||||
|
assert.Contains(t, page, "permanent")
|
||||||
|
assert.Contains(t, page, "12 ms")
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestAppsPage_ReferencesInventoryPlan ensures the copy on
|
// TestAppsPage_ReferencesInventoryPlan ensures the copy on
|
||||||
|
|||||||
@@ -51,6 +51,23 @@ func (s *Server) routes() {
|
|||||||
s.mux.Handle("POST /settings/rotate-token", s.requireSession(s.handleRotateToken))
|
s.mux.Handle("POST /settings/rotate-token", s.requireSession(s.handleRotateToken))
|
||||||
s.mux.Handle("GET /updates", s.requireSession(s.handleUpdates))
|
s.mux.Handle("GET /updates", s.requireSession(s.handleUpdates))
|
||||||
|
|
||||||
|
// Docker Compose discovery + management. The list/detail/logs pages
|
||||||
|
// are session-protected HTML; the lifecycle endpoints accept either a
|
||||||
|
// session (browser, CSRF-checked in the handler) or HTTP basic auth
|
||||||
|
// (scripting, on the /web/api/compose/* prefix). Compose is disabled
|
||||||
|
// per host via WORKER_COMPOSE_ENABLED=false; when off, the management
|
||||||
|
// handlers return 503 and the list page renders a banner.
|
||||||
|
s.mux.Handle("GET /compose", s.requireSession(s.handleComposeList))
|
||||||
|
s.mux.Handle("GET /compose/{project}", s.requireSession(s.handleComposeDetail))
|
||||||
|
s.mux.Handle("GET /compose/{project}/logs", s.requireSession(s.handleComposeLogsPage))
|
||||||
|
s.mux.Handle("POST /compose/{project}/{action}", s.requireSession(s.handleComposeProjectAction))
|
||||||
|
s.mux.Handle("POST /compose/{project}/service/{service}/{action}", s.requireSession(s.handleComposeServiceAction))
|
||||||
|
s.mux.Handle("GET /web/api/compose", s.requireSession(s.handleComposeAPIList))
|
||||||
|
s.mux.Handle("GET /web/api/compose/{project}", s.requireSession(s.handleComposeAPIDetail))
|
||||||
|
s.mux.Handle("GET /web/api/compose/{project}/logs", s.requireSession(s.handleComposeLogsAPI))
|
||||||
|
s.mux.Handle("POST /web/api/compose/{project}/{action}", s.requireSession(s.handleComposeProjectAction))
|
||||||
|
s.mux.Handle("POST /web/api/compose/{project}/service/{service}/{action}", s.requireSession(s.handleComposeServiceAction))
|
||||||
|
|
||||||
// Health endpoint for the cmd health subcommand and for the
|
// Health endpoint for the cmd health subcommand and for the
|
||||||
// operator to confirm the listener is up without going through the
|
// operator to confirm the listener is up without going through the
|
||||||
// login form. Returns 200 with a tiny body.
|
// login form. Returns 200 with a tiny body.
|
||||||
@@ -62,7 +79,6 @@ func (s *Server) routes() {
|
|||||||
// Both routes require a session (the worker webapp is single-tenant
|
// Both routes require a session (the worker webapp is single-tenant
|
||||||
// so every logged-in operator is effectively an admin).
|
// so every logged-in operator is effectively an admin).
|
||||||
s.mux.Handle("GET /web/api/cluster/status", s.requireSession(s.handleClusterStatus))
|
s.mux.Handle("GET /web/api/cluster/status", s.requireSession(s.handleClusterStatus))
|
||||||
s.mux.Handle("POST /web/api/cluster/apply-test-config", s.requireSession(s.handleClusterApplyTestConfig))
|
|
||||||
|
|
||||||
// Cross-worker peer status. The path is intentionally under
|
// Cross-worker peer status. The path is intentionally under
|
||||||
// /api/ (not /web/api/) so the basic-auth middleware does not
|
// /api/ (not /web/api/) so the basic-auth middleware does not
|
||||||
|
|||||||
@@ -80,16 +80,6 @@ type Config struct {
|
|||||||
BasicAuthLogin string
|
BasicAuthLogin string
|
||||||
BasicAuthPassword string
|
BasicAuthPassword string
|
||||||
|
|
||||||
// DebugClusterApply gates the /web/api/cluster/apply-test-config
|
|
||||||
// endpoint. When false (the default) the route is registered but
|
|
||||||
// the handler returns 404 so the endpoint is invisible in
|
|
||||||
// production. Operators who want to poke the cluster FSM during
|
|
||||||
// development set WORKER_CLUSTER_DEBUG_APPLY=true. The endpoint
|
|
||||||
// must NEVER be reachable in production — it appends hardcoded
|
|
||||||
// log entries to the Raft FSM without going through the real
|
|
||||||
// config-adoption producer.
|
|
||||||
DebugClusterApply bool
|
|
||||||
|
|
||||||
// ReleaseURL is the optional URL the worker polls to discover
|
// ReleaseURL is the optional URL the worker polls to discover
|
||||||
// the latest published version of the worker binary. When empty
|
// the latest published version of the worker binary. When empty
|
||||||
// the /updates page shows the placeholder "v1 (dev)". The URL
|
// the /updates page shows the placeholder "v1 (dev)". The URL
|
||||||
@@ -97,6 +87,14 @@ type Config struct {
|
|||||||
// "tag_name" field (GitHub release JSON is the canonical
|
// "tag_name" field (GitHub release JSON is the canonical
|
||||||
// shape). WORKER_RELEASE_URL sets this.
|
// shape). WORKER_RELEASE_URL sets this.
|
||||||
ReleaseURL string
|
ReleaseURL string
|
||||||
|
|
||||||
|
// ComposeEnabled turns the Docker Compose discovery + management
|
||||||
|
// subsystem on. Defaults to true (WORKER_COMPOSE_ENABLED=false to
|
||||||
|
// disable): the refresher no-ops when the Docker daemon is absent,
|
||||||
|
// so leaving it on is safe on non-Docker hosts. When enabled, the
|
||||||
|
// /compose page lists every Compose project on the host and the
|
||||||
|
// /web/api/compose/* endpoints drive up/down/stop/restart/pull.
|
||||||
|
ComposeEnabled bool
|
||||||
}
|
}
|
||||||
|
|
||||||
// ValidateBasicAuth enforces that WORKER_LOGIN and WORKER_PASSWORD
|
// ValidateBasicAuth enforces that WORKER_LOGIN and WORKER_PASSWORD
|
||||||
@@ -186,11 +184,26 @@ func ConfigFromEnv(env map[string]string, defaultDataDir string) (Config, error)
|
|||||||
if v := env["RSMON_WEBAPP_STORE_PATH"]; v != "" {
|
if v := env["RSMON_WEBAPP_STORE_PATH"]; v != "" {
|
||||||
cfg.StorePath = v
|
cfg.StorePath = v
|
||||||
}
|
}
|
||||||
cfg.DebugClusterApply = parseBool(env[envClusterDebugApply])
|
|
||||||
cfg.ReleaseURL = strings.TrimSpace(env[envReleaseURL])
|
cfg.ReleaseURL = strings.TrimSpace(env[envReleaseURL])
|
||||||
|
// Compose discovery defaults to on; only an explicit falsy literal
|
||||||
|
// disables it. An unset env var (empty) keeps the feature on.
|
||||||
|
cfg.ComposeEnabled = parseBoolTrueDefault(env[envComposeEnabled])
|
||||||
return cfg, nil
|
return cfg, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// parseBoolTrueDefault reports true for any value that is not an
|
||||||
|
// explicit falsy literal. The empty string (unset env var) is true,
|
||||||
|
// so a feature using this parser is on unless the operator opts out.
|
||||||
|
// This mirrors parseBool's literal set but inverts the default, which
|
||||||
|
// is what ComposeEnabled needs.
|
||||||
|
func parseBoolTrueDefault(v string) bool {
|
||||||
|
switch strings.ToLower(strings.TrimSpace(v)) {
|
||||||
|
case "false", "0", "no", "off":
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
// parseBool returns true for the strings "true", "1", "yes" (any
|
// parseBool returns true for the strings "true", "1", "yes" (any
|
||||||
// case, trimmed). Anything else is false. Used for opt-in feature
|
// case, trimmed). Anything else is false. Used for opt-in feature
|
||||||
// flags wired through env vars without dragging in a config-package
|
// flags wired through env vars without dragging in a config-package
|
||||||
@@ -212,7 +225,8 @@ func ConfigFromEnvOrDefault() Config {
|
|||||||
envWorkerHost, envWorkerPort, envWorkerURL, envWorkerLogin, envWorkerPassword,
|
envWorkerHost, envWorkerPort, envWorkerURL, envWorkerLogin, envWorkerPassword,
|
||||||
"RSMON_WEBAPP_DATA_DIR", "RSMON_WEBAPP_STORE_PATH",
|
"RSMON_WEBAPP_DATA_DIR", "RSMON_WEBAPP_STORE_PATH",
|
||||||
"WORKER_CLUSTER_ENABLED",
|
"WORKER_CLUSTER_ENABLED",
|
||||||
envClusterDebugApply, envReleaseURL,
|
envComposeEnabled,
|
||||||
|
envReleaseURL,
|
||||||
} {
|
} {
|
||||||
if v := os.Getenv(k); v != "" {
|
if v := os.Getenv(k); v != "" {
|
||||||
env[k] = v
|
env[k] = v
|
||||||
@@ -254,7 +268,6 @@ type Deps struct {
|
|||||||
// pulling in the raft package or bbolt.
|
// pulling in the raft package or bbolt.
|
||||||
type ClusterView interface {
|
type ClusterView interface {
|
||||||
Stats() ClusterStats
|
Stats() ClusterStats
|
||||||
ApplyTestConfig() (uint64, error)
|
|
||||||
ClusterID() string
|
ClusterID() string
|
||||||
LocalAddr() string
|
LocalAddr() string
|
||||||
}
|
}
|
||||||
@@ -320,9 +333,7 @@ type ResultRow struct {
|
|||||||
At time.Time
|
At time.Time
|
||||||
}
|
}
|
||||||
|
|
||||||
// NotificationRow is one row from the worker's in-memory notification
|
// NotificationRow is one row from the worker's in-memory notification ring.
|
||||||
// ring buffer. Phase 1 only emits selfcheck alerts; main-app-issued
|
|
||||||
// notifications still live in the main app's DB.
|
|
||||||
type NotificationRow struct {
|
type NotificationRow struct {
|
||||||
Kind string // "email", "telegram_private", "telegram_group"
|
Kind string // "email", "telegram_private", "telegram_group"
|
||||||
Channel string
|
Channel string
|
||||||
@@ -331,6 +342,11 @@ type NotificationRow struct {
|
|||||||
OK bool
|
OK bool
|
||||||
Error string
|
Error string
|
||||||
At time.Time
|
At time.Time
|
||||||
|
|
||||||
|
JobID string
|
||||||
|
Method string
|
||||||
|
Status string
|
||||||
|
DurationMs int
|
||||||
}
|
}
|
||||||
|
|
||||||
// Server is the local HTTP server for the worker webapp. It owns the
|
// Server is the local HTTP server for the worker webapp. It owns the
|
||||||
@@ -345,6 +361,7 @@ type Server struct {
|
|||||||
templates *Templates
|
templates *Templates
|
||||||
logBuffer *LogBuffer
|
logBuffer *LogBuffer
|
||||||
inventory *Inventory
|
inventory *Inventory
|
||||||
|
compose *ComposeRefresher
|
||||||
metrics *Metrics
|
metrics *Metrics
|
||||||
cluster ClusterView
|
cluster ClusterView
|
||||||
pruneStop chan struct{}
|
pruneStop chan struct{}
|
||||||
@@ -411,6 +428,7 @@ func New(cfg Config, deps *Deps) (*Server, error) { //nolint:gocritic // Config
|
|||||||
templates: tmpl,
|
templates: tmpl,
|
||||||
logBuffer: NewLogBuffer(5000),
|
logBuffer: NewLogBuffer(5000),
|
||||||
inventory: NewInventory(store, deps.Logger),
|
inventory: NewInventory(store, deps.Logger),
|
||||||
|
compose: NewComposeRefresher(cfg.ComposeEnabled, deps.Logger),
|
||||||
metrics: NewMetrics(),
|
metrics: NewMetrics(),
|
||||||
cluster: deps.Cluster,
|
cluster: deps.Cluster,
|
||||||
pruneStop: make(chan struct{}),
|
pruneStop: make(chan struct{}),
|
||||||
@@ -469,6 +487,10 @@ func (s *Server) SetCluster(c ClusterView) { s.cluster = c }
|
|||||||
// Cluster returns the attached cluster subsystem (or nil).
|
// Cluster returns the attached cluster subsystem (or nil).
|
||||||
func (s *Server) Cluster() ClusterView { return s.cluster }
|
func (s *Server) Cluster() ClusterView { return s.cluster }
|
||||||
|
|
||||||
|
// Compose returns the Compose discovery/management refresher so tests
|
||||||
|
// and the cmd binary can drive it (e.g. inject a fixture discovery).
|
||||||
|
func (s *Server) Compose() *ComposeRefresher { return s.compose }
|
||||||
|
|
||||||
// Close shuts down the HTTP listener, the prune goroutine, and the
|
// Close shuts down the HTTP listener, the prune goroutine, and the
|
||||||
// embedded store. Safe to call multiple times.
|
// embedded store. Safe to call multiple times.
|
||||||
func (s *Server) Close(ctx context.Context) error {
|
func (s *Server) Close(ctx context.Context) error {
|
||||||
@@ -481,6 +503,9 @@ func (s *Server) Close(ctx context.Context) error {
|
|||||||
close(s.pruneStop)
|
close(s.pruneStop)
|
||||||
}
|
}
|
||||||
s.pruneWG.Wait()
|
s.pruneWG.Wait()
|
||||||
|
if s.compose != nil {
|
||||||
|
s.compose.Stop()
|
||||||
|
}
|
||||||
if s.httpServer != nil {
|
if s.httpServer != nil {
|
||||||
shutdownCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
|
shutdownCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
|
||||||
defer cancel()
|
defer cancel()
|
||||||
@@ -502,6 +527,12 @@ func (s *Server) Start(ctx context.Context) error {
|
|||||||
s.pruneWG.Add(1)
|
s.pruneWG.Add(1)
|
||||||
go s.pruneLoop(ctx)
|
go s.pruneLoop(ctx)
|
||||||
|
|
||||||
|
// Start the Compose discovery loop alongside the listener. It no-ops
|
||||||
|
// when disabled or when Docker is absent, so it is always safe to
|
||||||
|
// start. The first refresh runs immediately so /compose has data on
|
||||||
|
// the first request.
|
||||||
|
s.compose.Start(ctx)
|
||||||
|
|
||||||
// Run ListenAndServe in a goroutine so we can race it against ctx.
|
// Run ListenAndServe in a goroutine so we can race it against ctx.
|
||||||
errCh := make(chan error, 1)
|
errCh := make(chan error, 1)
|
||||||
go func() {
|
go func() {
|
||||||
|
|||||||
@@ -88,37 +88,38 @@ func TestConfigFromEnvBasicAuthRejectsXOR(t *testing.T) {
|
|||||||
assert.Error(t, err, "XOR (password only) must be rejected")
|
assert.Error(t, err, "XOR (password only) must be rejected")
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestConfigFromEnvDebugClusterApply pins the default-off behavior
|
// TestConfigFromEnvComposeEnabled pins the true-default parsing of
|
||||||
// of the cluster-apply debug gate and verifies the env flag flips
|
// WORKER_COMPOSE_ENABLED: the feature is on when unset and only an
|
||||||
// it on. Production builds must not accidentally expose the
|
// explicit falsy literal turns it off. Regression guard for the
|
||||||
// endpoint, so the default is false.
|
// inverted-negation bug that made "true" disable the subsystem.
|
||||||
func TestConfigFromEnvDebugClusterApply(t *testing.T) {
|
func TestConfigFromEnvComposeEnabled(t *testing.T) {
|
||||||
cfg, err := ConfigFromEnv(map[string]string{}, t.TempDir())
|
cases := []struct {
|
||||||
require.NoError(t, err)
|
name string
|
||||||
assert.False(t, cfg.DebugClusterApply, "default must leave the debug flag off")
|
env string
|
||||||
|
want bool
|
||||||
cfg, err = ConfigFromEnv(map[string]string{
|
}{
|
||||||
"WORKER_CLUSTER_DEBUG_APPLY": "true",
|
{"unset empty", "", true},
|
||||||
}, t.TempDir())
|
{"explicit true", "true", true},
|
||||||
require.NoError(t, err)
|
{"explicit one", "1", true},
|
||||||
assert.True(t, cfg.DebugClusterApply)
|
{"explicit yes", "yes", true},
|
||||||
|
{"random non-falsy", "on", true},
|
||||||
// Other truthy spellings accepted.
|
{"explicit false", "false", false},
|
||||||
for _, v := range []string{"yes", "1", "TRUE", "YeS"} {
|
{"explicit zero", "0", false},
|
||||||
cfg, err = ConfigFromEnv(map[string]string{
|
{"explicit no", "no", false},
|
||||||
"WORKER_CLUSTER_DEBUG_APPLY": v,
|
{"explicit off", "off", false},
|
||||||
}, t.TempDir())
|
{"falsy with case", "FALSE", false},
|
||||||
require.NoError(t, err)
|
{"falsy with whitespace", " off ", false},
|
||||||
assert.True(t, cfg.DebugClusterApply, "must accept truthy value %q", v)
|
|
||||||
}
|
}
|
||||||
|
for _, c := range cases {
|
||||||
// Empty / unknown values stay false.
|
t.Run(c.name, func(t *testing.T) {
|
||||||
for _, v := range []string{"", "false", "0", "no"} {
|
env := map[string]string{}
|
||||||
cfg, err = ConfigFromEnv(map[string]string{
|
if c.env != "" {
|
||||||
"WORKER_CLUSTER_DEBUG_APPLY": v,
|
env[envComposeEnabled] = c.env
|
||||||
}, t.TempDir())
|
}
|
||||||
|
cfg, err := ConfigFromEnv(env, t.TempDir())
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
assert.False(t, cfg.DebugClusterApply, "must reject non-truthy value %q", v)
|
assert.Equal(t, c.want, cfg.ComposeEnabled)
|
||||||
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
44
internal/webapp/templates/compose.html
Обычный файл
44
internal/webapp/templates/compose.html
Обычный файл
@@ -0,0 +1,44 @@
|
|||||||
|
{{define "body"}}<section class="card">
|
||||||
|
<h1>Docker Compose projects</h1>
|
||||||
|
{{if not .Enabled}}
|
||||||
|
<p class="muted">Compose management is disabled (WORKER_COMPOSE_ENABLED=false). Re-enable it and restart the worker to see projects here.</p>
|
||||||
|
{{else}}
|
||||||
|
<p class="muted">Discovered from <code>docker compose ls</code> and <code>docker ps</code> labels, refreshed every 60s. Lifecycle actions (up/down/stop/restart/pull) run <code>docker compose</code> in each project's working directory.</p>
|
||||||
|
{{if .LastAt.IsZero}}
|
||||||
|
<p class="muted">No refresh yet — the first scan runs within 60s of start.</p>
|
||||||
|
{{else}}
|
||||||
|
<p class="muted">Last refresh: {{fmtTime .LastAt}}</p>
|
||||||
|
{{end}}
|
||||||
|
{{range .Snapshot.Errors}}
|
||||||
|
<p class="muted">⚠ {{.}}</p>
|
||||||
|
{{end}}
|
||||||
|
<table>
|
||||||
|
<thead>
|
||||||
|
<tr>
|
||||||
|
<th>Project</th>
|
||||||
|
<th>Status</th>
|
||||||
|
<th>Services</th>
|
||||||
|
<th>Containers</th>
|
||||||
|
<th>Running</th>
|
||||||
|
<th>Compose file</th>
|
||||||
|
<th></th>
|
||||||
|
</tr>
|
||||||
|
</thead>
|
||||||
|
<tbody>
|
||||||
|
{{range .Snapshot.Projects}}
|
||||||
|
<tr>
|
||||||
|
<td>{{.Name}}</td>
|
||||||
|
<td>{{.Status}}</td>
|
||||||
|
<td>{{.ServiceCount}}</td>
|
||||||
|
<td>{{.ContainerCount}}</td>
|
||||||
|
<td>{{.RunningCount}}</td>
|
||||||
|
<td>{{.ConfigFiles}}</td>
|
||||||
|
<td><a href="/compose/{{.Name}}">details</a></td>
|
||||||
|
</tr>
|
||||||
|
{{else}}
|
||||||
|
<tr><td colspan="7" class="muted">No Compose projects discovered.</td></tr>
|
||||||
|
{{end}}
|
||||||
|
</tbody>
|
||||||
|
</table>
|
||||||
|
{{end}}
|
||||||
|
</section>{{end}}
|
||||||
81
internal/webapp/templates/compose_detail.html
Обычный файл
81
internal/webapp/templates/compose_detail.html
Обычный файл
@@ -0,0 +1,81 @@
|
|||||||
|
{{define "body"}}<section class="card">
|
||||||
|
<h1>{{.Project.Name}}</h1>
|
||||||
|
<p class="muted">Working dir: <code>{{.Project.WorkingDir}}</code>{{if .Project.ConfigFiles}} · compose file: <code>{{.Project.ConfigFiles}}</code>{{end}} · last refresh: {{fmtTime .LastAt}}</p>
|
||||||
|
|
||||||
|
{{if .ActionMessage}}
|
||||||
|
<p class="{{if .ActionOK}}muted{{else}}muted{{end}}">{{if .ActionOK}}✓{{else}}⚠{{end}} {{.ActionMessage}}</p>
|
||||||
|
{{end}}
|
||||||
|
|
||||||
|
<h2>Project actions</h2>
|
||||||
|
<div class="cards">
|
||||||
|
<form action="/compose/{{.Project.Name}}/up" method="post"><input type="hidden" name="csrf_token" value="{{.CSRFToken}}"><button type="submit">Up</button></form>
|
||||||
|
<form action="/compose/{{.Project.Name}}/restart" method="post"><input type="hidden" name="csrf_token" value="{{.CSRFToken}}"><button type="submit">Restart</button></form>
|
||||||
|
<form action="/compose/{{.Project.Name}}/pull" method="post"><input type="hidden" name="csrf_token" value="{{.CSRFToken}}"><button type="submit">Pull</button></form>
|
||||||
|
<form action="/compose/{{.Project.Name}}/stop" method="post"><input type="hidden" name="csrf_token" value="{{.CSRFToken}}"><button type="submit">Stop</button></form>
|
||||||
|
<form action="/compose/{{.Project.Name}}/down" method="post"><input type="hidden" name="csrf_token" value="{{.CSRFToken}}"><button type="submit">Down</button></form>
|
||||||
|
<a href="/compose/{{.Project.Name}}/logs"><button type="button">Logs</button></a>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<h2>Services ({{.Project.ServiceCount}})</h2>
|
||||||
|
<table>
|
||||||
|
<thead><tr><th>Service</th><th>Container</th><th>Image</th><th>State</th><th>Status</th><th>Health</th><th>PID</th><th>Ports</th><th>Actions</th></tr></thead>
|
||||||
|
<tbody>
|
||||||
|
{{$root := .}}
|
||||||
|
{{range .Project.Services}}
|
||||||
|
<tr>
|
||||||
|
<td>{{.Name}}</td>
|
||||||
|
<td>{{range .Containers}}{{.Name}}<br>{{end}}</td>
|
||||||
|
<td>{{range .Containers}}{{.Image}}<br>{{end}}</td>
|
||||||
|
<td>{{range .Containers}}{{.State}}<br>{{end}}</td>
|
||||||
|
<td>{{range .Containers}}{{.Status}}<br>{{end}}</td>
|
||||||
|
<td>{{range .Containers}}{{.Health}}<br>{{end}}</td>
|
||||||
|
<td>{{range .Containers}}{{.PID}}<br>{{end}}</td>
|
||||||
|
<td>{{range .Containers}}{{range .Ports}}{{.HostPort}}<br>{{end}}{{end}}</td>
|
||||||
|
<td>
|
||||||
|
<form action="/compose/{{$root.Project.Name}}/service/{{.Name}}/up" method="post" style="display:inline"><input type="hidden" name="csrf_token" value="{{$root.CSRFToken}}"><button type="submit">up</button></form>
|
||||||
|
<form action="/compose/{{$root.Project.Name}}/service/{{.Name}}/restart" method="post" style="display:inline"><input type="hidden" name="csrf_token" value="{{$root.CSRFToken}}"><button type="submit">restart</button></form>
|
||||||
|
<form action="/compose/{{$root.Project.Name}}/service/{{.Name}}/stop" method="post" style="display:inline"><input type="hidden" name="csrf_token" value="{{$root.CSRFToken}}"><button type="submit">stop</button></form>
|
||||||
|
</td>
|
||||||
|
</tr>
|
||||||
|
{{else}}
|
||||||
|
<tr><td colspan="9" class="muted">No services discovered for this project.</td></tr>
|
||||||
|
{{end}}
|
||||||
|
</tbody>
|
||||||
|
</table>
|
||||||
|
|
||||||
|
{{if .Project.GroupedMounts}}
|
||||||
|
<h2>Mounts</h2>
|
||||||
|
<table>
|
||||||
|
<thead><tr><th>Source</th><th>Destination</th><th>Type</th><th>Used by</th></tr></thead>
|
||||||
|
<tbody>
|
||||||
|
{{range .Project.GroupedMounts}}
|
||||||
|
<tr>
|
||||||
|
<td>{{.Source}}</td>
|
||||||
|
<td>{{.Destination}}</td>
|
||||||
|
<td>{{.Type}}</td>
|
||||||
|
<td>{{.SharedMountSummary}}</td>
|
||||||
|
</tr>
|
||||||
|
{{end}}
|
||||||
|
</tbody>
|
||||||
|
</table>
|
||||||
|
{{end}}
|
||||||
|
|
||||||
|
{{if .Project.AllTraefikRoutes}}
|
||||||
|
<h2>Traefik routes</h2>
|
||||||
|
<table>
|
||||||
|
<thead><tr><th>Router</th><th>Service</th><th>Container</th><th>Hostnames</th><th>Prefixes</th><th>Rule</th></tr></thead>
|
||||||
|
<tbody>
|
||||||
|
{{range .Project.AllTraefikRoutes}}
|
||||||
|
<tr>
|
||||||
|
<td>{{.RouterName}}</td>
|
||||||
|
<td>{{.Service}}</td>
|
||||||
|
<td>{{.Container}}</td>
|
||||||
|
<td>{{range .Hostnames}}{{.}}<br>{{end}}</td>
|
||||||
|
<td>{{range .PathPrefixes}}{{.}}<br>{{end}}</td>
|
||||||
|
<td><code>{{.Rule}}</code></td>
|
||||||
|
</tr>
|
||||||
|
{{end}}
|
||||||
|
</tbody>
|
||||||
|
</table>
|
||||||
|
{{end}}
|
||||||
|
</section>{{end}}
|
||||||
6
internal/webapp/templates/compose_logs.html
Обычный файл
6
internal/webapp/templates/compose_logs.html
Обычный файл
@@ -0,0 +1,6 @@
|
|||||||
|
{{define "body"}}<section class="card">
|
||||||
|
<h1>Compose logs: {{.Project}}</h1>
|
||||||
|
<p class="muted">Last 300 lines from <code>docker compose logs --no-color --tail 300</code>. Streaming tails are not supported in the web UI yet.</p>
|
||||||
|
<pre class="logs">{{if .Output}}{{.Output}}{{else}}<span class="muted">(no output)</span>{{end}}</pre>
|
||||||
|
<p><a href="/compose/{{.Project}}">← back to {{.Project}}</a></p>
|
||||||
|
</section>{{end}}
|
||||||
@@ -14,6 +14,7 @@
|
|||||||
<nav>
|
<nav>
|
||||||
<a href="/overview">Overview</a>
|
<a href="/overview">Overview</a>
|
||||||
<a href="/apps">Apps</a>
|
<a href="/apps">Apps</a>
|
||||||
|
<a href="/compose">Compose</a>
|
||||||
<a href="/checks">Checks</a>
|
<a href="/checks">Checks</a>
|
||||||
<a href="/notifications">Notifications</a>
|
<a href="/notifications">Notifications</a>
|
||||||
<a href="/logs">Logs</a>
|
<a href="/logs">Logs</a>
|
||||||
|
|||||||
@@ -1,15 +1,15 @@
|
|||||||
{{define "body"}}<section class="card">
|
{{define "body"}}<section class="card">
|
||||||
<h1>Recent notifications</h1>
|
<h1>Recent notifications</h1>
|
||||||
<p class="muted">Last 50 notifications emitted by this worker (Phase 1: selfcheck alerts only).</p>
|
<p class="muted">Last 50 notification attempts emitted by this worker.</p>
|
||||||
<p><button type="button" disabled title="{{.ResendTooltip}}">Resend selected</button>
|
<p><button type="button" disabled title="{{.ResendTooltip}}">Resend selected</button>
|
||||||
<span class="muted">{{.ResendTooltip}}</span></p>
|
<span class="muted">{{.ResendTooltip}}</span></p>
|
||||||
<table>
|
<table>
|
||||||
<thead>
|
<thead>
|
||||||
<tr>
|
<tr>
|
||||||
<th>Channel</th>
|
<th>Type</th>
|
||||||
<th>Subject</th>
|
<th>Details</th>
|
||||||
<th>Body</th>
|
|
||||||
<th>Status</th>
|
<th>Status</th>
|
||||||
|
<th>Duration</th>
|
||||||
<th>Error</th>
|
<th>Error</th>
|
||||||
<th>When</th>
|
<th>When</th>
|
||||||
</tr>
|
</tr>
|
||||||
@@ -17,11 +17,19 @@
|
|||||||
<tbody>
|
<tbody>
|
||||||
{{range .Rows}}
|
{{range .Rows}}
|
||||||
<tr>
|
<tr>
|
||||||
<td>{{.Channel}} ({{.Kind}})</td>
|
{{if .JobID}}
|
||||||
<td>{{.Subject}}</td>
|
<td>delegated</td>
|
||||||
<td><code>{{.Body}}</code></td>
|
<td>job <code>{{.JobID}}</code>, {{.Method}}</td>
|
||||||
|
<td>{{.Status}}</td>
|
||||||
|
<td>{{.DurationMs}} ms</td>
|
||||||
|
<td></td>
|
||||||
|
{{else}}
|
||||||
|
<td>selfcheck</td>
|
||||||
|
<td>{{.Channel}} ({{.Kind}}): {{.Subject}} <code>{{.Body}}</code></td>
|
||||||
<td>{{if .OK}}<span class="ok">delivered</span>{{else}}<span class="error">failed</span>{{end}}</td>
|
<td>{{if .OK}}<span class="ok">delivered</span>{{else}}<span class="error">failed</span>{{end}}</td>
|
||||||
|
<td>-</td>
|
||||||
<td>{{.Error}}</td>
|
<td>{{.Error}}</td>
|
||||||
|
{{end}}
|
||||||
<td>{{fmtTime .At}}</td>
|
<td>{{fmtTime .At}}</td>
|
||||||
</tr>
|
</tr>
|
||||||
{{else}}
|
{{else}}
|
||||||
|
|||||||
@@ -4,10 +4,15 @@ package wire
|
|||||||
import "encoding/json"
|
import "encoding/json"
|
||||||
|
|
||||||
// RegisterRequest is sent to the control plane registration API.
|
// RegisterRequest is sent to the control plane registration API.
|
||||||
|
// PublicURL is the canonical advertised origin (scheme and authority
|
||||||
|
// only, see docs/public-endpoint-and-identity.md). The legacy URL field
|
||||||
|
// remains on the wire for the bounded migration from WORKER_URL; new
|
||||||
|
// senders populate PublicURL and old control planes keep reading URL.
|
||||||
type RegisterRequest struct {
|
type RegisterRequest struct {
|
||||||
WorkerID string `json:"worker_id" binding:"required"`
|
WorkerID string `json:"worker_id" binding:"required"`
|
||||||
RegionCode string `json:"region_code" binding:"required"`
|
RegionCode string `json:"region_code" binding:"required"`
|
||||||
Version string `json:"version"`
|
Version string `json:"version"`
|
||||||
|
PublicURL string `json:"public_url,omitempty"`
|
||||||
URL string `json:"url,omitempty"`
|
URL string `json:"url,omitempty"`
|
||||||
Capabilities []string `json:"capabilities"`
|
Capabilities []string `json:"capabilities"`
|
||||||
TaskEnvelope bool `json:"task_envelope"`
|
TaskEnvelope bool `json:"task_envelope"`
|
||||||
@@ -20,6 +25,17 @@ type RegisterResponse struct {
|
|||||||
WorkerID string `json:"worker_id"`
|
WorkerID string `json:"worker_id"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type BootstrapRequest struct {
|
||||||
|
WorkerID string `json:"worker_id"`
|
||||||
|
BootstrapToken string `json:"bootstrap_token"`
|
||||||
|
}
|
||||||
|
type BootstrapResponse struct {
|
||||||
|
AuthToken string `json:"auth_token"`
|
||||||
|
WorkerID string `json:"worker_id"`
|
||||||
|
ConfigVerificationKey string `json:"config_verification_key"`
|
||||||
|
SigningKeyID string `json:"signing_key_id"`
|
||||||
|
}
|
||||||
|
|
||||||
// HeartbeatRequest is sent periodically by workers to indicate liveness.
|
// HeartbeatRequest is sent periodically by workers to indicate liveness.
|
||||||
type HeartbeatRequest struct {
|
type HeartbeatRequest struct {
|
||||||
ActiveChecks int `json:"active_checks"`
|
ActiveChecks int `json:"active_checks"`
|
||||||
@@ -39,6 +55,7 @@ type CheckJob struct {
|
|||||||
URL *string `json:"url"`
|
URL *string `json:"url"`
|
||||||
Interval int `json:"interval"`
|
Interval int `json:"interval"`
|
||||||
Settings json.RawMessage `json:"settings"` // CheckSettings JSON
|
Settings json.RawMessage `json:"settings"` // CheckSettings JSON
|
||||||
|
AccountID int64 `json:"account_id,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// JobsResponse contains a batch of check jobs for a worker
|
// JobsResponse contains a batch of check jobs for a worker
|
||||||
@@ -203,6 +220,9 @@ type PeerInfo struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// WorkerInit is sent by the control plane after websocket authentication.
|
// WorkerInit is sent by the control plane after websocket authentication.
|
||||||
|
// PublicURL is the canonical advertised origin accepted by the control
|
||||||
|
// plane; URL is the legacy wire field kept for the bounded migration.
|
||||||
|
// A worker must prefer PublicURL when it is non-empty.
|
||||||
type WorkerInit struct {
|
type WorkerInit struct {
|
||||||
WorkerID string `json:"worker_id"`
|
WorkerID string `json:"worker_id"`
|
||||||
RegionCode string `json:"region_code"`
|
RegionCode string `json:"region_code"`
|
||||||
@@ -211,6 +231,7 @@ type WorkerInit struct {
|
|||||||
NotificationMethods []string `json:"notification_methods,omitempty"`
|
NotificationMethods []string `json:"notification_methods,omitempty"`
|
||||||
NotificationAccounts []int64 `json:"notification_accounts,omitempty"`
|
NotificationAccounts []int64 `json:"notification_accounts,omitempty"`
|
||||||
Concurrency int `json:"concurrency"`
|
Concurrency int `json:"concurrency"`
|
||||||
|
PublicURL string `json:"public_url,omitempty"`
|
||||||
URL string `json:"url,omitempty"`
|
URL string `json:"url,omitempty"`
|
||||||
ServerID *int64 `json:"server_id,omitempty"`
|
ServerID *int64 `json:"server_id,omitempty"`
|
||||||
LLMs []LLMConfig `json:"llms,omitempty"`
|
LLMs []LLMConfig `json:"llms,omitempty"`
|
||||||
@@ -222,6 +243,15 @@ type WorkerInit struct {
|
|||||||
// the control plane; a worker that receives an empty list treats
|
// the control plane; a worker that receives an empty list treats
|
||||||
// the selfcheck as a single-node decision (no peer polling).
|
// the selfcheck as a single-node decision (no peer polling).
|
||||||
Peers []PeerInfo `json:"peers,omitempty"`
|
Peers []PeerInfo `json:"peers,omitempty"`
|
||||||
|
AccountID *int64 `json:"account_id,omitempty"`
|
||||||
|
ConfigVersion int64 `json:"config_version,omitempty"`
|
||||||
|
IssuedAt string `json:"issued_at,omitempty"`
|
||||||
|
ExpiresAt string `json:"expires_at,omitempty"`
|
||||||
|
CredentialSetHash string `json:"credential_set_hash,omitempty"`
|
||||||
|
Signature string `json:"signature,omitempty"`
|
||||||
|
SigningKeyID string `json:"signing_key_id,omitempty"`
|
||||||
|
RotateToken string `json:"rotate_token,omitempty"`
|
||||||
|
RotationID string `json:"rotation_id,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// TaskEnvelope is the task frame the control plane sends on the worker
|
// TaskEnvelope is the task frame the control plane sends on the worker
|
||||||
@@ -315,5 +345,20 @@ type WorkerMessage struct {
|
|||||||
NotificationResult *NotificationResultReport `json:"notification_result,omitempty"`
|
NotificationResult *NotificationResultReport `json:"notification_result,omitempty"`
|
||||||
ServerMetric *ServerMetricReport `json:"server_metric,omitempty"`
|
ServerMetric *ServerMetricReport `json:"server_metric,omitempty"`
|
||||||
Heartbeat *HeartbeatRequest `json:"heartbeat,omitempty"`
|
Heartbeat *HeartbeatRequest `json:"heartbeat,omitempty"`
|
||||||
|
RotationAck *RotationAck `json:"rotation_ack,omitempty"`
|
||||||
|
StaleLease *StaleLeaseReport `json:"stale_lease,omitempty"`
|
||||||
|
StaleLeaseAck *StaleLeaseAck `json:"stale_lease_ack,omitempty"`
|
||||||
Error string `json:"error,omitempty"`
|
Error string `json:"error,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type RotationAck struct {
|
||||||
|
RotationID string `json:"rotation_id"`
|
||||||
|
}
|
||||||
|
type StaleLeaseReport struct {
|
||||||
|
JobID string `json:"job_id"`
|
||||||
|
LeaseToken string `json:"lease_token"`
|
||||||
|
}
|
||||||
|
type StaleLeaseAck struct {
|
||||||
|
JobID string `json:"job_id"`
|
||||||
|
LeaseToken string `json:"lease_token"`
|
||||||
|
}
|
||||||
|
|||||||
@@ -151,6 +151,82 @@ func TestRegisterRequest_URLOmittedWhenEmpty(t *testing.T) {
|
|||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
assert.False(t, strings.Contains(string(data), `"url"`),
|
assert.False(t, strings.Contains(string(data), `"url"`),
|
||||||
"empty URL must be omitted from the register payload, got %s", string(data))
|
"empty URL must be omitted from the register payload, got %s", string(data))
|
||||||
|
assert.False(t, strings.Contains(string(data), `"public_url"`),
|
||||||
|
"empty PublicURL must be omitted from the register payload, got %s", string(data))
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestRegisterRequest_PublicURLRoundTrip verifies the canonical public_url
|
||||||
|
// wire field serializes and deserializes alongside the legacy url field.
|
||||||
|
func TestRegisterRequest_PublicURLRoundTrip(t *testing.T) {
|
||||||
|
req := RegisterRequest{
|
||||||
|
WorkerID: "worker-eu-1",
|
||||||
|
RegionCode: "eu",
|
||||||
|
Version: "v1",
|
||||||
|
PublicURL: "https://worker-eu.example.com",
|
||||||
|
}
|
||||||
|
|
||||||
|
data, err := json.Marshal(req)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Contains(t, string(data), `"public_url":"https://worker-eu.example.com"`,
|
||||||
|
"PublicURL must serialize as the top-level public_url field, got %s", string(data))
|
||||||
|
|
||||||
|
var decoded RegisterRequest
|
||||||
|
require.NoError(t, json.Unmarshal(data, &decoded))
|
||||||
|
assert.Equal(t, "https://worker-eu.example.com", decoded.PublicURL)
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestWorkerInit_PublicURLRoundTrip covers the canonical public_url field
|
||||||
|
// on the init/config frame.
|
||||||
|
func TestWorkerInit_PublicURLRoundTrip(t *testing.T) {
|
||||||
|
init := WorkerInit{
|
||||||
|
WorkerID: "worker-1",
|
||||||
|
RegionCode: "ru",
|
||||||
|
Version: "v1",
|
||||||
|
Concurrency: 4,
|
||||||
|
PublicURL: "https://worker-eu.example.com",
|
||||||
|
}
|
||||||
|
|
||||||
|
data, err := json.Marshal(init)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Contains(t, string(data), `"public_url":"https://worker-eu.example.com"`,
|
||||||
|
"PublicURL must serialize as a top-level public_url field, got %s", string(data))
|
||||||
|
|
||||||
|
var decoded WorkerInit
|
||||||
|
require.NoError(t, json.Unmarshal(data, &decoded))
|
||||||
|
assert.Equal(t, "https://worker-eu.example.com", decoded.PublicURL)
|
||||||
|
assert.Equal(t, "", decoded.URL,
|
||||||
|
"legacy URL field must stay empty when only public_url is set")
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestWorkerInit_LegacyURLFieldStillDecodes keeps the bounded-migration
|
||||||
|
// contract: a control plane that still sends the legacy url field must
|
||||||
|
// remain wire-compatible with the new worker struct.
|
||||||
|
func TestWorkerInit_LegacyURLFieldStillDecodes(t *testing.T) {
|
||||||
|
data := []byte(`{"worker_id":"worker-1","region_code":"ru","concurrency":4,"url":"https://legacy.example.com"}`)
|
||||||
|
var decoded WorkerInit
|
||||||
|
require.NoError(t, json.Unmarshal(data, &decoded))
|
||||||
|
assert.Equal(t, "https://legacy.example.com", decoded.URL)
|
||||||
|
assert.Equal(t, "", decoded.PublicURL)
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestWorkerInit_PublicURLAndLegacyCoexist ensures a frame that carries
|
||||||
|
// both fields keeps both on the wire for old and new control planes.
|
||||||
|
func TestWorkerInit_PublicURLAndLegacyCoexist(t *testing.T) {
|
||||||
|
init := WorkerInit{
|
||||||
|
WorkerID: "w-1",
|
||||||
|
PublicURL: "https://canonical.example.com",
|
||||||
|
URL: "https://legacy.example.com",
|
||||||
|
}
|
||||||
|
data, err := json.Marshal(init)
|
||||||
|
require.NoError(t, err)
|
||||||
|
out := string(data)
|
||||||
|
assert.Contains(t, out, `"public_url":"https://canonical.example.com"`)
|
||||||
|
assert.Contains(t, out, `"url":"https://legacy.example.com"`)
|
||||||
|
|
||||||
|
var decoded WorkerInit
|
||||||
|
require.NoError(t, json.Unmarshal(data, &decoded))
|
||||||
|
assert.Equal(t, "https://canonical.example.com", decoded.PublicURL)
|
||||||
|
assert.Equal(t, "https://legacy.example.com", decoded.URL)
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestWorkerInit_NotificationCapabilitiesRoundTrip ensures the
|
// TestWorkerInit_NotificationCapabilitiesRoundTrip ensures the
|
||||||
|
|||||||
@@ -72,11 +72,9 @@ func TestClusterSmokeStats(t *testing.T) {
|
|||||||
assert.Equal(t, addr, c.LocalAddr())
|
assert.Equal(t, addr, c.LocalAddr())
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestClusterApplyTestConfig_Smoke verifies the ApplyTestConfig helper
|
// TestClusterTestConfig_Smoke verifies the test-only helper commits a
|
||||||
// commits a config.adopt entry and the FSM reflects the new version.
|
// config.adopt entry and the FSM reflects the new version.
|
||||||
// This is the function the webapp admin endpoint and the CLI flag
|
func TestClusterTestConfig_Smoke(t *testing.T) {
|
||||||
// both go through.
|
|
||||||
func TestClusterApplyTestConfig_Smoke(t *testing.T) {
|
|
||||||
addr := pickPort(t)
|
addr := pickPort(t)
|
||||||
dataDir := t.TempDir()
|
dataDir := t.TempDir()
|
||||||
creds := HTTPCreds{Login: "alice", Password: "secret"}
|
creds := HTTPCreds{Login: "alice", Password: "secret"}
|
||||||
@@ -108,8 +106,8 @@ func TestClusterApplyTestConfig_Smoke(t *testing.T) {
|
|||||||
require.Equal(t, raft.Leader, c.Raft().State(),
|
require.Equal(t, raft.Leader, c.Raft().State(),
|
||||||
"smoke test requires the local node to be leader")
|
"smoke test requires the local node to be leader")
|
||||||
|
|
||||||
check := DefaultDebugCriticalCheck()
|
check := defaultTestCriticalCheck()
|
||||||
idx, err := c.ApplyTestConfig(&check)
|
idx, err := c.applyTestConfig(&check)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
assert.NotZero(t, idx, "applied index must be non-zero")
|
assert.NotZero(t, idx, "applied index must be non-zero")
|
||||||
|
|
||||||
@@ -164,11 +162,11 @@ func TestClusterStats_FSMFieldsOnFreshCluster(t *testing.T) {
|
|||||||
"newly created FSM defaults Partition.State to 'steady'")
|
"newly created FSM defaults Partition.State to 'steady'")
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestApplyTestConfig_NonLeaderErrors pins the precondition that the
|
// TestTestConfig_NonLeaderErrors pins the precondition that the
|
||||||
// helper refuses to submit an entry on a non-leader (the raft library
|
// helper refuses to submit an entry on a non-leader (the raft library
|
||||||
// would reject the apply anyway, but we want the failure to be
|
// would reject the apply anyway, but we want the failure to be
|
||||||
// deterministic and informative).
|
// deterministic and informative).
|
||||||
func TestApplyTestConfig_NonLeaderErrors(t *testing.T) {
|
func TestTestConfig_NonLeaderErrors(t *testing.T) {
|
||||||
// Three-node fixture so we have a clear "not the leader" node.
|
// Three-node fixture so we have a clear "not the leader" node.
|
||||||
if testing.Short() {
|
if testing.Short() {
|
||||||
t.Skip("3-node smoke skipped in -short mode")
|
t.Skip("3-node smoke skipped in -short mode")
|
||||||
@@ -187,9 +185,9 @@ func TestApplyTestConfig_NonLeaderErrors(t *testing.T) {
|
|||||||
require.NotNil(t, leader)
|
require.NotNil(t, leader)
|
||||||
require.NotNil(t, follower)
|
require.NotNil(t, follower)
|
||||||
|
|
||||||
check := DefaultDebugCriticalCheck()
|
check := defaultTestCriticalCheck()
|
||||||
_, err := follower.ApplyTestConfig(&check)
|
_, err := follower.applyTestConfig(&check)
|
||||||
require.Error(t, err, "non-leader must refuse ApplyTestConfig")
|
require.Error(t, err, "non-leader must refuse test config application")
|
||||||
assert.Contains(t, strings.ToLower(err.Error()), "not leader")
|
assert.Contains(t, strings.ToLower(err.Error()), "not leader")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -2,7 +2,6 @@ package workercluster
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"encoding/json"
|
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
@@ -325,63 +324,6 @@ func (c *Cluster) Snapshot() error {
|
|||||||
return r.Snapshot().Error()
|
return r.Snapshot().Error()
|
||||||
}
|
}
|
||||||
|
|
||||||
// DefaultDebugCriticalCheck returns the hardcoded CriticalCheckConfig
|
|
||||||
// the cluster admin debug endpoint and the
|
|
||||||
// --cluster-debug-apply-test-config CLI flag apply. A fresh Epoch is
|
|
||||||
// stamped on every call so repeated applies produce distinct entries
|
|
||||||
// (handy for verifying replication timing).
|
|
||||||
func DefaultDebugCriticalCheck() CriticalCheckConfig {
|
|
||||||
return CriticalCheckConfig{
|
|
||||||
ID: 9999,
|
|
||||||
Kind: "distributed_critical",
|
|
||||||
IntervalS: 30,
|
|
||||||
Target: "http://example.com",
|
|
||||||
Epoch: time.Now().UTC().UnixNano(),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// ApplyTestConfig submits a hardcoded config.adopt log entry with the
|
|
||||||
// supplied CriticalCheckConfig. Returns the applied log index. This
|
|
||||||
// is a debug convenience used by the e2e script and the
|
|
||||||
// --cluster-debug-apply-test-config CLI flag; production code should
|
|
||||||
// build entries from the real signed-config-adoption producer
|
|
||||||
// (Phase-N work).
|
|
||||||
//
|
|
||||||
// DEBUG: this exists only so the e2e shell script can verify FSM
|
|
||||||
// replication without a real producer wired in.
|
|
||||||
//
|
|
||||||
// TODO(phase-N): remove once the real producer lands.
|
|
||||||
func (c *Cluster) ApplyTestConfig(check *CriticalCheckConfig) (uint64, error) {
|
|
||||||
c.mu.Lock()
|
|
||||||
r := c.raft
|
|
||||||
c.mu.Unlock()
|
|
||||||
if r == nil {
|
|
||||||
return 0, errors.New("workercluster: not started")
|
|
||||||
}
|
|
||||||
if r.State() != raft.Leader {
|
|
||||||
return 0, errors.New("workercluster: not leader; submit on the leader")
|
|
||||||
}
|
|
||||||
|
|
||||||
payload := ConfigAdoptPayload{
|
|
||||||
Version: 1,
|
|
||||||
Actor: c.opts.NodeID,
|
|
||||||
Checks: []CriticalCheckConfig{*check},
|
|
||||||
}
|
|
||||||
raw, err := json.Marshal(payload)
|
|
||||||
if err != nil {
|
|
||||||
return 0, fmt.Errorf("workercluster: encode payload: %w", err)
|
|
||||||
}
|
|
||||||
entry, err := EncodeEntry(&Entry{Type: EntryConfigAdopt, Adopted: raw})
|
|
||||||
if err != nil {
|
|
||||||
return 0, fmt.Errorf("workercluster: encode entry: %w", err)
|
|
||||||
}
|
|
||||||
fut := r.Apply(entry, 10*time.Second)
|
|
||||||
if err := fut.Error(); err != nil {
|
|
||||||
return 0, fmt.Errorf("workercluster: apply test config: %w", err)
|
|
||||||
}
|
|
||||||
return fut.Index(), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// ClusterID returns the NodeID this cluster was constructed with. It
|
// ClusterID returns the NodeID this cluster was constructed with. It
|
||||||
// is exposed so HTTP handlers can label status responses with a
|
// is exposed so HTTP handlers can label status responses with a
|
||||||
// stable identifier even when the raft library's own State() reports
|
// stable identifier even when the raft library's own State() reports
|
||||||
|
|||||||
Некоторые файлы не были показаны из-за слишком большого количества измененных файлов Показать больше
Ссылка в новой задаче
Block a user