diff --git a/.github/workflows/test-ssh.yml b/.github/workflows/test-ssh.yml new file mode 100644 index 0000000..a8870e6 --- /dev/null +++ b/.github/workflows/test-ssh.yml @@ -0,0 +1,94 @@ +name: SSH Source-Install E2E + +# Work package 5 of docs/source-installation.md: run the Alpine/Ubuntu/Arch +# OpenSSH source-install E2E matrix (make test-ssh) in CI. +# +# Trust boundary. This job builds and runs privileged Docker containers from +# repository code (the distro fixtures execute the checked-out source), so it +# runs ONLY on trusted refs: pushes to the default branch (master) and manual +# workflow_dispatch. It deliberately does NOT trigger on pull_request: a PR +# can carry untrusted code into a privileged Docker environment, and Gitea's +# fork-PR read-only token clamp does not change what the containers can do on +# the runner host. The ordinary unit `make test` CI run (ci.yml) stays +# Docker-free and still covers pull requests. +# +# Runner requirement. The job inherits whatever Docker access `docker.yml` +# already relies on. The harness dials fixture SSH ports published on the +# Docker daemon loopback, so the runner must expose Docker with loopback port +# publishing reachable from the job; scripts/ci/test-ssh.sh fails fast when +# that is not the case. +# +# External network: fixture images (alpine from the reg.rsxx.ru mirror, +# ubuntu/arch overridable via RSMON_TEST_IMAGE_*), the pinned Go toolchain +# from go.dev, the public source repo from rocketgit.ru, and distro package +# repos are fetched live. Operators can pin a resolver for flaky CI DNS via +# the RSMON_TEST_DOCKER_DNS repository variable (comma-separated nameservers). + +on: + push: + branches: + - master + workflow_dispatch: + +# Per-ref scoping: master pushes cancel a superseded in-flight run on the same +# ref instead of stacking; a manual dispatch on another branch has its own +# group and never cancels the master run. +concurrency: + group: test-ssh-${{ gitea.ref }} + cancel-in-progress: true + +# Least privilege: the job only needs to read the repository (actions/checkout +# and the harness's SSH-free operations). contents:read is supported by Gitea +# Actions (the GITEA_TOKEN scope for code/releases). +permissions: + contents: read + +jobs: + test-ssh: + name: Alpine/Ubuntu/Arch source-install E2E + runs-on: ubuntu-latest + timeout-minutes: 90 + env: + # All empty by default (harness defaults / Docker embedded DNS). + # Override in repo/organization variables: + # RSMON_TEST_IMAGE_ALPINE / _UBUNTU / _ARCH - pin a mirror or a + # specific distro snapshot for the fixture base image; + # RSMON_TEST_DOCKER_DNS - comma-separated nameservers applied as + # `docker run --dns ...` for flaky CI resolvers. + RSMON_TEST_IMAGE_ALPINE: ${{ vars.RSMON_TEST_IMAGE_ALPINE }} + RSMON_TEST_IMAGE_UBUNTU: ${{ vars.RSMON_TEST_IMAGE_UBUNTU }} + RSMON_TEST_IMAGE_ARCH: ${{ vars.RSMON_TEST_IMAGE_ARCH }} + RSMON_TEST_DOCKER_DNS: ${{ vars.RSMON_TEST_DOCKER_DNS }} + steps: + # Actions are pinned to immutable full commit SHAs (not moving tags). + # Verified 2026-08-13 against the GitHub API that the commit the tag + # points to is a commit object: + # actions/checkout@v4 -> 11d5960a326750d5838078e36cf38b85af677262 + # actions/setup-go@v5 -> 40f1582b2485089dde7abd97c1529aa768e1baff + # Other workflows (ci.yml, docker.yml) still use moving tags; see the + # repo-wide convention note in docs/source-installation.md. + - name: Check out code + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 + + - name: Set up Go + uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff + with: + go-version: '1.26.x' + # Caches the Go module/build cache used to compile the harness test + # binary. Safe: only public dependencies, no credentials. + cache: true + + - name: Run Alpine/Ubuntu/Arch OpenSSH source-install E2E + run: bash scripts/ci/test-ssh.sh + + # Belt-and-suspenders: the harness and test-ssh.sh already tear down + # everything they create; this guarantees a killed job leaves no + # rsmon-worker-test-* resources behind on the runner. Filters are + # anchored to the harness's own prefix and image repository so cleanup + # never touches a shared base image or unrelated resources. + - name: Clean up leftover harness resources + if: always() + run: | + docker ps -aq --filter "name=^rsmon-worker-test-" 2>/dev/null | xargs -r docker rm -f >/dev/null 2>&1 || true + docker network ls -q --filter "name=^rsmon-worker-test-" 2>/dev/null | xargs -r docker network rm >/dev/null 2>&1 || true + docker images -q --filter "reference=rsmon-worker-test/*" 2>/dev/null | xargs -r docker image rm -f >/dev/null 2>&1 || true diff --git a/docs/changelog.md b/docs/changelog.md index 57eff81..ddd65ff 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -1,5 +1,50 @@ # Changelog +## 2026-08-13 + +### Source-install E2E in CI (work package 5) + +- Added `.github/workflows/test-ssh.yml`, a Gitea Actions workflow that runs + the Alpine/Ubuntu/Arch Docker/OpenSSH source-install matrix (`make test-ssh`) + on pushes to `master` and on manual `workflow_dispatch` only, separate from + the Docker-free unit CI. It does not trigger on `pull_request`: the distro + fixtures execute the checked-out code inside privileged Docker, so untrusted + PR code must never run there automatically. It bounds the job with + `timeout-minutes: 90` (the go-test `-timeout 60m` stays in place), scopes + concurrency per ref (`test-ssh-${{ gitea.ref }}`), declares + `permissions: contents: read`, uploads no artifacts, and cleans up on every + path. +- Pinned the two actions to immutable full commit SHAs (verified against the + GitHub API): `actions/checkout@v4` -> + `11d5960a326750d5838078e36cf38b85af677262` and `actions/setup-go@v5` -> + `40f1582b2485089dde7abd97c1529aa768e1baff`. The repo-wide convention still + leaves `ci.yml`/`docker.yml` on moving tags (accepted, documented risk); see + `docs/source-installation.md`. +- Wired `RSMON_TEST_IMAGE_ALPINE` / `RSMON_TEST_IMAGE_UBUNTU` / + `RSMON_TEST_IMAGE_ARCH` and `RSMON_TEST_DOCKER_DNS` repository variables + (all empty by default) so CI can pin per-fixture mirror/snapshot images and + a resolver for flaky CI DNS. +- Added `scripts/ci/test-ssh.sh`: preflights Docker and the harness's + loopback port-publishing requirement with a tiny `docker run -p + 127.0.0.1::22` probe (fails fast with an actionable message plus + diagnostics/fix options instead of a 60m timeout on an unsupported runner), + then runs `make test-ssh` and traps `EXIT` to remove every leftover + `rsmon-worker-test-*` container/network/image tag. Cleanup filters are + anchored to the harness's own prefix/repository so they never touch a shared + base image. The workflow adds an `if: always()` cleanup step as a + belt-and-suspenders so a killed job never leaves test material on the runner. +- External network is fetched live by design (go.dev toolchain, rocketgit.ru + source clone, distro repos); operators can pin a resolver via the + `RSMON_TEST_DOCKER_DNS` repository variable (comma-separated nameservers, + applied as `docker run --dns ...`) and mirror/snapshot overrides via + `RSMON_TEST_IMAGE_ALPINE` / `RSMON_TEST_IMAGE_UBUNTU` / + `RSMON_TEST_IMAGE_ARCH`. +- Fixed a history-dependent test fragility: `TestSourceInstallDirtyCheckoutPreservesStaging` + and the rollback test's build-failure step dirty the tracked tree by + appending a marker line to `Makefile` instead of `git checkout master~1 -- + Makefile`, which silently stopped dirtying the tree once the last commit did + not touch that file. + ## 2026-08-12 ### Source-install hardening review diff --git a/docs/implementation-roadmap.md b/docs/implementation-roadmap.md index e1428a4..acfd303 100644 --- a/docs/implementation-roadmap.md +++ b/docs/implementation-roadmap.md @@ -32,9 +32,9 @@ Source-install foundations landed: SHA-256-verified Go toolchain, clone/update of the public repo, resolved branch/commit record, and a staging build. Running service/config is not touched (source-install work package 3); -- [ ] atomic service activation, rollback, and failure-preservation tests over +- [x] atomic service activation, rollback, and failure-preservation tests over SSH (source-install work package 4); -- [ ] run the full source-install E2E matrix in CI (source-install work +- [x] run the full source-install E2E matrix in CI (source-install work package 5). Gate: a push publishes `sha-<12>` and `latest` manifests for both platforms, diff --git a/docs/source-installation.md b/docs/source-installation.md index 228959e..5aab9dc 100644 --- a/docs/source-installation.md +++ b/docs/source-installation.md @@ -2,7 +2,8 @@ ## Status -In progress. Work packages 1-4 are implemented: +In progress. Work packages 1-5 are implemented (work package 5 is the +Docker/OpenSSH E2E matrix wired into CI): - `internal/installer/harness` builds and runs real OpenSSH containers for Alpine, Ubuntu, and Arch, waits for real network readiness, captures @@ -246,6 +247,91 @@ for activation and is supplied with `--token`/`--token-file` or `--env-file` (prefer the file options; direct flags expose the value through the process list). +## Work Package 5: Alpine/Ubuntu/Arch E2E In CI + +Work package 5 runs the full Docker/OpenSSH E2E matrix on pushes to +`master` and on manual `workflow_dispatch` runs. It is a separate Gitea +Actions workflow (`.github/workflows/test-ssh.yml`) so the ordinary unit +CI run stays Docker-free; the job executes `make test-ssh` through +`scripts/ci/test-ssh.sh`. + +CI behavior: + +- **Trust boundary and triggers.** The workflow runs *only* on pushes to + `master` (the default branch) and on manual `workflow_dispatch`. It + deliberately does **not** trigger on `pull_request`: the distro fixtures + execute the checked-out repository code inside privileged Docker + containers, so an untrusted PR must never reach the runner's Docker + surface automatically. Gitea's read-only token clamp for fork PRs does + not limit what containers can do on the runner host, so PR coverage is + left to the Docker-free unit CI (`ci.yml`) and to manual dispatch after + a human reviews the change. +- **Concurrency is scoped per ref.** `concurrency.group: + test-ssh-${{ gitea.ref }}` gives master pushes their own group (a newer + master push cancels a superseded in-flight master run instead of + stacking) and gives a manual dispatch on another branch its own group so + it never cancels the master run. Gitea Actions evaluates the expression, + and `gitea.ref` is the same documented context the repo's `docker.yml` + already uses. +- **Least privilege.** The workflow declares `permissions: contents: read` + (supported by Gitea Actions as the `GITEA_TOKEN` scope for + code/releases), so the job's token can only read the repository; the + workflow never writes, pushes, or publishes. +- **Action revisions.** The two actions this workflow uses are pinned to + immutable full commit SHAs (not moving tags): `actions/checkout@v4` -> + `11d5960a326750d5838078e36cf38b85af677262` and `actions/setup-go@v5` -> + `40f1582b2485089dde7abd97c1529aa768e1baff` (verified 2026-08-13 against + the GitHub API that each tag points to a commit object). The repo-wide + convention still leaves `ci.yml` and `docker.yml` on moving tags + (`@v4`, `@v5`, `@v3`, `@v6`); that is an accepted, documented risk: a + tag move can change behavior without a workflow diff. New workflows + should pin SHAs like this one; migrating the existing workflows is a + separate change. +- **Runner requirement.** The harness dials fixture SSH ports published on + the Docker daemon's `127.0.0.1`, so the job must share the daemon's + loopback (a host-mode runner or a job container with host networking). + `scripts/ci/test-ssh.sh` probes this with a tiny `docker run -p + 127.0.0.1::22` round trip *before* the matrix and fails fast with a + clear, actionable message (including diagnostics and fix options) + instead of after a 60m go-test timeout. +- **Bounding and timeout.** The workflow sets `timeout-minutes: 90` and + the existing `make test-ssh` go-test `-timeout 60m` stays in place, so + the whole job is hard-bounded even under slow networks or downloads. +- **Cleanup.** The harness already tears down every container, network, + per-instance fixture image tag, and temp dir on success and failure. + `scripts/ci/test-ssh.sh` additionally traps `EXIT` to remove any + leftover `rsmon-worker-test-*` resource (container, network, or the + per-instance `rsmon-worker-test/-:local` image tag), and + the workflow adds an `if: always()` step that does the same even when the + script itself is killed. All filters are anchored to the harness's own + prefix and image repository (`name=^rsmon-worker-test-`, + `reference=rsmon-worker-test/*`), so cleanup never touches a shared base + image (`reg.rsxx.ru/library/alpine:3`, `ubuntu:24.04`, `archlinux:latest`) + or an unrelated resource. +- **Mirrors and overrides.** The Alpine fixture already uses the + `reg.rsxx.ru/library/alpine:3` mirror. Ubuntu and Arch have no mirror yet + and default to Docker Hub. The workflow wires the + `RSMON_TEST_IMAGE_ALPINE`, `RSMON_TEST_IMAGE_UBUNTU`, and + `RSMON_TEST_IMAGE_ARCH` repository variables (empty by default) so an + operator can pin a mirror or a specific distro snapshot per fixture, and + the `RSMON_TEST_DOCKER_DNS` repository variable (comma-separated + nameservers, applied as `docker run --dns ...`) to pin a resolver for + flaky CI DNS. External network (go.dev toolchain download, the public + rocketgit.ru source clone, distro package repos) is fetched live by + design. +- **Artifacts and secrets.** The workflow uploads no artifacts: the go-test + log stays in the runner's job log and nothing private (test key, + `known_hosts`, env files) is retained on the runner or published. The + fixture containers receive only the fixed `e2e-activation-test-token` + and the bundled test-only key, never a real worker token. +- **Caching.** `actions/setup-go` caches the Go module/build cache used to + compile the harness test binary (public dependencies only). Fixture and + toolchain downloads are not cached because they run inside disposable + distro containers; reruns rebuild them cleanly. + +Work package 5 does not change how the source installer behaves. It only +adds a CI surface for the existing acceptance tests. + ## Docker OpenSSH Test Harness Adapt the real-network pattern from `/data/_swap/sshkeymanager`: start an @@ -400,7 +486,7 @@ RSMON_TEST_DOCKER=1 go test -v -count=1 -timeout 30m ./internal/installer/harnes env, data dir, and init service definition installed atomically; process and `/healthz` verified; activation/start/health failures restore the prior install; reruns keep exactly one service). -- [ ] 5. Add Alpine, Ubuntu, and Arch network E2E tests to CI. +- [x] 5. Add Alpine, Ubuntu, and Arch network E2E tests to CI. - [ ] 6. Add CentOS-family support. - [ ] 7. Plan native Windows service and macOS launchd installers separately. @@ -419,7 +505,7 @@ RSMON_TEST_DOCKER=1 go test -v -count=1 -timeout 30m ./internal/installer/harnes the previous installation (build/checkout, activation, start, and health failure reruns on the Alpine fixture all restore the prior binary, env, and running service). -- [ ] CI uses approved registry mirrors and cleans every test container/network. +- [x] CI uses approved registry mirrors and cleans every test container/network. ## Verified Test Evidence (work packages 1-4) @@ -483,3 +569,27 @@ Recorded 2026-08-12 from `make test-ssh` (Docker Engine 29.7.1): tests above rather than against a real init. The stub tools simulate unit state and command flow, not real systemd/OpenRC unit semantics; a real init-system smoke test on a booted host remains a follow-up. + +Work package 5 (CI) evidence, recorded 2026-08-13: + +- `.github/workflows/test-ssh.yml` runs `scripts/ci/test-ssh.sh` on pushes + to `master` and on manual `workflow_dispatch` only (never on + `pull_request`, because the fixtures execute the checked-out code inside + privileged Docker). Concurrency is scoped per ref + (`test-ssh-${{ gitea.ref }}`), the job token is `contents: read`, and + the script's loopback port-publishing probe passes on a host Docker + daemon while its `EXIT` cleanup leaves zero leftover containers, + networks, or fixture image tags. +- A full local `make test-ssh` run (the exact command the CI job executes) + passes the complete harness suite: `TestHarnessFixtures` (alpine/ubuntu/ + arch), host-key mismatch and stability, teardown, `TestSourceInstallFixtures` + (work package 3 staging on all three distros), `TestSourceInstallActivationFixtures` + (work package 4 activation on all three), activation failure rollback, + `--no-start`, and the dirty-checkout preservation test. Pinned + `RSMON_TEST_DOCKER_DNS=1.1.1.1` was used because the local network's + default resolvers intermittently time out on `rocketgit.ru`. +- `TestSourceInstallDirtyCheckoutPreservesStaging` and the rollback test's + build-failure step now dirty the tracked tree deterministically (appending + a marker line to `Makefile`) instead of `git checkout master~1 -- Makefile`, + which depended on the last two commits differing and silently stopped + dirtying the tree once a commit did not touch that file. diff --git a/internal/installer/harness/integration_activation_test.go b/internal/installer/harness/integration_activation_test.go index b8c8531..4accd38 100644 --- a/internal/installer/harness/integration_activation_test.go +++ b/internal/installer/harness/integration_activation_test.go @@ -258,8 +258,11 @@ func TestSourceInstallActivationFailureRollback(t *testing.T) { // 1. Build failure preserves the prior install. // A dirty tracked tree makes the rerun's checkout refuse before - // the build, so neither staging nor the service changes. - if _, err := RunCommand(client, "git -C "+shellQuote(res.Plan.BuildDir)+" checkout -q master~1 -- Makefile"); err != nil { + // the build, so neither staging nor the service changes. Appending + // a marker is deterministic regardless of the cloned history (a + // `git checkout master~1 -- ` would depend on whether that + // commit differs from the resolved one). + if _, err := RunCommand(client, "printf '\\n# rsmon-worker dirty-tree marker\\n' >> "+shellQuote(res.Plan.BuildDir)+"/Makefile"); err != nil { t.Fatalf("dirty the working tree: %v", err) } before := installedBinarySHA(t, client, a) diff --git a/internal/installer/harness/integration_test.go b/internal/installer/harness/integration_test.go index 407019c..4b4f58b 100644 --- a/internal/installer/harness/integration_test.go +++ b/internal/installer/harness/integration_test.go @@ -466,8 +466,12 @@ func TestSourceInstallDirtyCheckoutPreservesStaging(t *testing.T) { defer client.Close() //nolint:errcheck // Dirty a tracked file so the rerun's checkout refuses to proceed. - // Makefile differs between master and master~1 (unlike go.mod). - if _, err := RunCommand(client, "git -C "+shellQuote(res.Plan.BuildDir)+" checkout -q master~1 -- Makefile"); err != nil { + // Appending a marker is deterministic regardless of the cloned history + // (a `git checkout master~1 -- ` would depend on whether that + // commit differs from the resolved one), and an appended line to a + // tracked file is always an unstaged modification `git diff --quiet` + // catches. + if _, err := RunCommand(client, "printf '\\n# rsmon-worker dirty-tree marker\\n' >> "+shellQuote(res.Plan.BuildDir)+"/Makefile"); err != nil { t.Fatalf("dirty the working tree: %v", err) } beforeBinary, err := RunCommand(client, "sha256sum "+shellQuote(res.StageBinary)) diff --git a/scripts/ci/test-ssh.sh b/scripts/ci/test-ssh.sh new file mode 100755 index 0000000..24b1cb7 --- /dev/null +++ b/scripts/ci/test-ssh.sh @@ -0,0 +1,78 @@ +#!/usr/bin/env bash +# CI entrypoint for the source-install Docker/OpenSSH E2E matrix +# (work package 5, docs/source-installation.md). +# +# Runs `make test-ssh` (Alpine/Ubuntu/Arch real OpenSSH fixtures over the +# public network) inside a Docker-capable CI job. It: +# - fails fast with a clear message when Docker is unreachable, because the +# harness dials fixture SSH ports published on the daemon's loopback; +# - runs the full fixture matrix with make's 60m go-test timeout; +# - cleans up every leftover test container/network/fixture-image tag on +# success AND failure, so an aborted job never leaks test material on the +# runner. +# +# It never deletes a shared base image: only rsmon-worker-test-* resources and +# the per-instance rsmon-worker-test/-:local tags the harness +# owns are removed. +set -euo pipefail + +# cleanup removes only resources the harness owns. Runs on every exit path. +cleanup() { + # Containers, then networks (which need no attached containers). Filters + # are anchored to the harness's own prefix so a resource that merely + # contains the substring is never touched. + docker ps -aq --filter "name=^rsmon-worker-test-" 2>/dev/null \ + | xargs -r docker rm -f >/dev/null 2>&1 || true + docker network ls -q --filter "name=^rsmon-worker-test-" 2>/dev/null \ + | xargs -r docker network rm >/dev/null 2>&1 || true + # Per-instance fixture image tags only; never a shared base image. + docker images -q --filter "reference=rsmon-worker-test/*" 2>/dev/null \ + | xargs -r docker image rm -f >/dev/null 2>&1 || true +} +trap cleanup EXIT + +if ! command -v docker >/dev/null 2>&1; then + echo "test-ssh CI: docker CLI not found; this job needs a Docker-capable runner" >&2 + exit 1 +fi + +if ! docker info >/dev/null 2>&1; then + echo "test-ssh CI: Docker daemon is not reachable; this job needs a working Docker daemon" >&2 + exit 1 +fi + +echo "test-ssh CI: Docker $(docker version --format '{{.Server.Version}}' 2>/dev/null || echo n/a) ready" + +# The harness publishes fixture SSH ports to the daemon's 127.0.0.1 and +# dials them from the test process, so the job must share the daemon's +# loopback. Probe that cheaply (no Go, no fixture build) before the heavy +# matrix so a misconfigured runner fails fast instead of after 60m. +cid="$(docker run -d --name rsmon-worker-test-preflight \ + -p 127.0.0.1::22 reg.rsxx.ru/library/alpine:3 sleep 300)" +port="$(docker port "$cid" 22/tcp 2>/dev/null | sed -n 's#.*:##p' | head -n1)" +if [ -z "$port" ] || ! (exec 3<>"/dev/tcp/127.0.0.1/${port}") 2>/dev/null; then + cat >&2 < from the test process, so the job and +the Docker daemon must share a loopback network namespace. + +Fix (pick one): + 1. Run this workflow on a host-mode runner (act_runner job executing on the + host with the local Docker daemon), OR + 2. Give the job container host networking with the Docker socket mounted + (e.g. container options: --network host plus /var/run/docker.sock), OR + 3. Run \`make test-ssh\` directly on a machine with a local Docker daemon. + +Diagnostics: + docker server: $(docker version --format '{{.Server.Version}}' 2>/dev/null || echo unknown) + published port: ${port:-none} +EOF + exit 1 +fi +exec 3>&- 2>/dev/null || true + +echo "test-ssh CI: loopback port publishing verified; running the Alpine/Ubuntu/Arch matrix" + +make test-ssh