ci(installer): run trusted SSH E2E matrix

.github/workflows/test-ssh.yml

@@ -0,0 +1,94 @@
+name: SSH Source-Install E2E
+
+# Work package 5 of docs/source-installation.md: run the Alpine/Ubuntu/Arch
+# OpenSSH source-install E2E matrix (make test-ssh) in CI.
+#
+# Trust boundary. This job builds and runs privileged Docker containers from
+# repository code (the distro fixtures execute the checked-out source), so it
+# runs ONLY on trusted refs: pushes to the default branch (master) and manual
+# workflow_dispatch. It deliberately does NOT trigger on pull_request: a PR
+# can carry untrusted code into a privileged Docker environment, and Gitea's
+# fork-PR read-only token clamp does not change what the containers can do on
+# the runner host. The ordinary unit `make test` CI run (ci.yml) stays
+# Docker-free and still covers pull requests.
+#
+# Runner requirement. The job inherits whatever Docker access `docker.yml`
+# already relies on. The harness dials fixture SSH ports published on the
+# Docker daemon loopback, so the runner must expose Docker with loopback port
+# publishing reachable from the job; scripts/ci/test-ssh.sh fails fast when
+# that is not the case.
+#
+# External network: fixture images (alpine from the reg.rsxx.ru mirror,
+# ubuntu/arch overridable via RSMON_TEST_IMAGE_*), the pinned Go toolchain
+# from go.dev, the public source repo from rocketgit.ru, and distro package
+# repos are fetched live. Operators can pin a resolver for flaky CI DNS via
+# the RSMON_TEST_DOCKER_DNS repository variable (comma-separated nameservers).
+
+on:
+  push:
+    branches:
+      - master
+  workflow_dispatch:
+
+# Per-ref scoping: master pushes cancel a superseded in-flight run on the same
+# ref instead of stacking; a manual dispatch on another branch has its own
+# group and never cancels the master run.
+concurrency:
+  group: test-ssh-${{ gitea.ref }}
+  cancel-in-progress: true
+
+# Least privilege: the job only needs to read the repository (actions/checkout
+# and the harness's SSH-free operations). contents:read is supported by Gitea
+# Actions (the GITEA_TOKEN scope for code/releases).
+permissions:
+  contents: read
+
+jobs:
+  test-ssh:
+    name: Alpine/Ubuntu/Arch source-install E2E
+    runs-on: ubuntu-latest
+    timeout-minutes: 90
+    env:
+      # All empty by default (harness defaults / Docker embedded DNS).
+      # Override in repo/organization variables:
+      #   RSMON_TEST_IMAGE_ALPINE / _UBUNTU / _ARCH - pin a mirror or a
+      #     specific distro snapshot for the fixture base image;
+      #   RSMON_TEST_DOCKER_DNS - comma-separated nameservers applied as
+      #     `docker run --dns ...` for flaky CI resolvers.
+      RSMON_TEST_IMAGE_ALPINE: ${{ vars.RSMON_TEST_IMAGE_ALPINE }}
+      RSMON_TEST_IMAGE_UBUNTU: ${{ vars.RSMON_TEST_IMAGE_UBUNTU }}
+      RSMON_TEST_IMAGE_ARCH: ${{ vars.RSMON_TEST_IMAGE_ARCH }}
+      RSMON_TEST_DOCKER_DNS: ${{ vars.RSMON_TEST_DOCKER_DNS }}
+    steps:
+      # Actions are pinned to immutable full commit SHAs (not moving tags).
+      # Verified 2026-08-13 against the GitHub API that the commit the tag
+      # points to is a commit object:
+      #   actions/checkout@v4 -> 11d5960a326750d5838078e36cf38b85af677262
+      #   actions/setup-go@v5 -> 40f1582b2485089dde7abd97c1529aa768e1baff
+      # Other workflows (ci.yml, docker.yml) still use moving tags; see the
+      # repo-wide convention note in docs/source-installation.md.
+      - name: Check out code
+        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262
+
+      - name: Set up Go
+        uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff
+        with:
+          go-version: '1.26.x'
+          # Caches the Go module/build cache used to compile the harness test
+          # binary. Safe: only public dependencies, no credentials.
+          cache: true
+
+      - name: Run Alpine/Ubuntu/Arch OpenSSH source-install E2E
+        run: bash scripts/ci/test-ssh.sh
+
+      # Belt-and-suspenders: the harness and test-ssh.sh already tear down
+      # everything they create; this guarantees a killed job leaves no
+      # rsmon-worker-test-* resources behind on the runner. Filters are
+      # anchored to the harness's own prefix and image repository so cleanup
+      # never touches a shared base image or unrelated resources.
+      - name: Clean up leftover harness resources
+        if: always()
+        run: |
+          docker ps -aq --filter "name=^rsmon-worker-test-" 2>/dev/null | xargs -r docker rm -f >/dev/null 2>&1 || true
+          docker network ls -q --filter "name=^rsmon-worker-test-" 2>/dev/null | xargs -r docker network rm >/dev/null 2>&1 || true
+          docker images -q --filter "reference=rsmon-worker-test/*" 2>/dev/null | xargs -r docker image rm -f >/dev/null 2>&1 || true

docs/changelog.md

@@ -1,5 +1,50 @@
 # Changelog
 
+## 2026-08-13
+
+### Source-install E2E in CI (work package 5)
+
+- Added `.github/workflows/test-ssh.yml`, a Gitea Actions workflow that runs
+  the Alpine/Ubuntu/Arch Docker/OpenSSH source-install matrix (`make test-ssh`)
+  on pushes to `master` and on manual `workflow_dispatch` only, separate from
+  the Docker-free unit CI. It does not trigger on `pull_request`: the distro
+  fixtures execute the checked-out code inside privileged Docker, so untrusted
+  PR code must never run there automatically. It bounds the job with
+  `timeout-minutes: 90` (the go-test `-timeout 60m` stays in place), scopes
+  concurrency per ref (`test-ssh-${{ gitea.ref }}`), declares
+  `permissions: contents: read`, uploads no artifacts, and cleans up on every
+  path.
+- Pinned the two actions to immutable full commit SHAs (verified against the
+  GitHub API): `actions/checkout@v4` ->
+  `11d5960a326750d5838078e36cf38b85af677262` and `actions/setup-go@v5` ->
+  `40f1582b2485089dde7abd97c1529aa768e1baff`. The repo-wide convention still
+  leaves `ci.yml`/`docker.yml` on moving tags (accepted, documented risk); see
+  `docs/source-installation.md`.
+- Wired `RSMON_TEST_IMAGE_ALPINE` / `RSMON_TEST_IMAGE_UBUNTU` /
+  `RSMON_TEST_IMAGE_ARCH` and `RSMON_TEST_DOCKER_DNS` repository variables
+  (all empty by default) so CI can pin per-fixture mirror/snapshot images and
+  a resolver for flaky CI DNS.
+- Added `scripts/ci/test-ssh.sh`: preflights Docker and the harness's
+  loopback port-publishing requirement with a tiny `docker run -p
+  127.0.0.1::22` probe (fails fast with an actionable message plus
+  diagnostics/fix options instead of a 60m timeout on an unsupported runner),
+  then runs `make test-ssh` and traps `EXIT` to remove every leftover
+  `rsmon-worker-test-*` container/network/image tag. Cleanup filters are
+  anchored to the harness's own prefix/repository so they never touch a shared
+  base image. The workflow adds an `if: always()` cleanup step as a
+  belt-and-suspenders so a killed job never leaves test material on the runner.
+- External network is fetched live by design (go.dev toolchain, rocketgit.ru
+  source clone, distro repos); operators can pin a resolver via the
+  `RSMON_TEST_DOCKER_DNS` repository variable (comma-separated nameservers,
+  applied as `docker run --dns ...`) and mirror/snapshot overrides via
+  `RSMON_TEST_IMAGE_ALPINE` / `RSMON_TEST_IMAGE_UBUNTU` /
+  `RSMON_TEST_IMAGE_ARCH`.
+- Fixed a history-dependent test fragility: `TestSourceInstallDirtyCheckoutPreservesStaging`
+  and the rollback test's build-failure step dirty the tracked tree by
+  appending a marker line to `Makefile` instead of `git checkout master~1 --
+  Makefile`, which silently stopped dirtying the tree once the last commit did
+  not touch that file.
+
 ## 2026-08-12
 
 ### Source-install hardening review

docs/implementation-roadmap.md

@@ -32,9 +32,9 @@ Source-install foundations landed:
       SHA-256-verified Go toolchain, clone/update of the public repo, resolved
       branch/commit record, and a staging build. Running service/config is not
       touched (source-install work package 3);
-- [ ] atomic service activation, rollback, and failure-preservation tests over
+- [x] atomic service activation, rollback, and failure-preservation tests over
       SSH (source-install work package 4);
-- [ ] run the full source-install E2E matrix in CI (source-install work
+- [x] run the full source-install E2E matrix in CI (source-install work
       package 5).
 
 Gate: a push publishes `sha-<12>` and `latest` manifests for both platforms,

docs/source-installation.md

@@ -2,7 +2,8 @@
 
 ## Status
 
-In progress. Work packages 1-4 are implemented:
+In progress. Work packages 1-5 are implemented (work package 5 is the
+Docker/OpenSSH E2E matrix wired into CI):
 
 - `internal/installer/harness` builds and runs real OpenSSH containers
   for Alpine, Ubuntu, and Arch, waits for real network readiness, captures
@@ -246,6 +247,91 @@ for activation and is supplied with `--token`/`--token-file` or
 `--env-file` (prefer the file options; direct flags expose the value
 through the process list).
 
+## Work Package 5: Alpine/Ubuntu/Arch E2E In CI
+
+Work package 5 runs the full Docker/OpenSSH E2E matrix on pushes to
+`master` and on manual `workflow_dispatch` runs. It is a separate Gitea
+Actions workflow (`.github/workflows/test-ssh.yml`) so the ordinary unit
+CI run stays Docker-free; the job executes `make test-ssh` through
+`scripts/ci/test-ssh.sh`.
+
+CI behavior:
+
+- **Trust boundary and triggers.** The workflow runs *only* on pushes to
+  `master` (the default branch) and on manual `workflow_dispatch`. It
+  deliberately does **not** trigger on `pull_request`: the distro fixtures
+  execute the checked-out repository code inside privileged Docker
+  containers, so an untrusted PR must never reach the runner's Docker
+  surface automatically. Gitea's read-only token clamp for fork PRs does
+  not limit what containers can do on the runner host, so PR coverage is
+  left to the Docker-free unit CI (`ci.yml`) and to manual dispatch after
+  a human reviews the change.
+- **Concurrency is scoped per ref.** `concurrency.group:
+  test-ssh-${{ gitea.ref }}` gives master pushes their own group (a newer
+  master push cancels a superseded in-flight master run instead of
+  stacking) and gives a manual dispatch on another branch its own group so
+  it never cancels the master run. Gitea Actions evaluates the expression,
+  and `gitea.ref` is the same documented context the repo's `docker.yml`
+  already uses.
+- **Least privilege.** The workflow declares `permissions: contents: read`
+  (supported by Gitea Actions as the `GITEA_TOKEN` scope for
+  code/releases), so the job's token can only read the repository; the
+  workflow never writes, pushes, or publishes.
+- **Action revisions.** The two actions this workflow uses are pinned to
+  immutable full commit SHAs (not moving tags): `actions/checkout@v4` ->
+  `11d5960a326750d5838078e36cf38b85af677262` and `actions/setup-go@v5` ->
+  `40f1582b2485089dde7abd97c1529aa768e1baff` (verified 2026-08-13 against
+  the GitHub API that each tag points to a commit object). The repo-wide
+  convention still leaves `ci.yml` and `docker.yml` on moving tags
+  (`@v4`, `@v5`, `@v3`, `@v6`); that is an accepted, documented risk: a
+  tag move can change behavior without a workflow diff. New workflows
+  should pin SHAs like this one; migrating the existing workflows is a
+  separate change.
+- **Runner requirement.** The harness dials fixture SSH ports published on
+  the Docker daemon's `127.0.0.1`, so the job must share the daemon's
+  loopback (a host-mode runner or a job container with host networking).
+  `scripts/ci/test-ssh.sh` probes this with a tiny `docker run -p
+  127.0.0.1::22` round trip *before* the matrix and fails fast with a
+  clear, actionable message (including diagnostics and fix options)
+  instead of after a 60m go-test timeout.
+- **Bounding and timeout.** The workflow sets `timeout-minutes: 90` and
+  the existing `make test-ssh` go-test `-timeout 60m` stays in place, so
+  the whole job is hard-bounded even under slow networks or downloads.
+- **Cleanup.** The harness already tears down every container, network,
+  per-instance fixture image tag, and temp dir on success and failure.
+  `scripts/ci/test-ssh.sh` additionally traps `EXIT` to remove any
+  leftover `rsmon-worker-test-*` resource (container, network, or the
+  per-instance `rsmon-worker-test/<fixture>-<suffix>:local` image tag), and
+  the workflow adds an `if: always()` step that does the same even when the
+  script itself is killed. All filters are anchored to the harness's own
+  prefix and image repository (`name=^rsmon-worker-test-`,
+  `reference=rsmon-worker-test/*`), so cleanup never touches a shared base
+  image (`reg.rsxx.ru/library/alpine:3`, `ubuntu:24.04`, `archlinux:latest`)
+  or an unrelated resource.
+- **Mirrors and overrides.** The Alpine fixture already uses the
+  `reg.rsxx.ru/library/alpine:3` mirror. Ubuntu and Arch have no mirror yet
+  and default to Docker Hub. The workflow wires the
+  `RSMON_TEST_IMAGE_ALPINE`, `RSMON_TEST_IMAGE_UBUNTU`, and
+  `RSMON_TEST_IMAGE_ARCH` repository variables (empty by default) so an
+  operator can pin a mirror or a specific distro snapshot per fixture, and
+  the `RSMON_TEST_DOCKER_DNS` repository variable (comma-separated
+  nameservers, applied as `docker run --dns ...`) to pin a resolver for
+  flaky CI DNS. External network (go.dev toolchain download, the public
+  rocketgit.ru source clone, distro package repos) is fetched live by
+  design.
+- **Artifacts and secrets.** The workflow uploads no artifacts: the go-test
+  log stays in the runner's job log and nothing private (test key,
+  `known_hosts`, env files) is retained on the runner or published. The
+  fixture containers receive only the fixed `e2e-activation-test-token`
+  and the bundled test-only key, never a real worker token.
+- **Caching.** `actions/setup-go` caches the Go module/build cache used to
+  compile the harness test binary (public dependencies only). Fixture and
+  toolchain downloads are not cached because they run inside disposable
+  distro containers; reruns rebuild them cleanly.
+
+Work package 5 does not change how the source installer behaves. It only
+adds a CI surface for the existing acceptance tests.
+
 ## Docker OpenSSH Test Harness
 
 Adapt the real-network pattern from `/data/_swap/sshkeymanager`: start an
@@ -400,7 +486,7 @@ RSMON_TEST_DOCKER=1 go test -v -count=1 -timeout 30m ./internal/installer/harnes
       env, data dir, and init service definition installed atomically; process
       and `/healthz` verified; activation/start/health failures restore the
       prior install; reruns keep exactly one service).
-- [ ] 5. Add Alpine, Ubuntu, and Arch network E2E tests to CI.
+- [x] 5. Add Alpine, Ubuntu, and Arch network E2E tests to CI.
 - [ ] 6. Add CentOS-family support.
 - [ ] 7. Plan native Windows service and macOS launchd installers separately.
 
@@ -419,7 +505,7 @@ RSMON_TEST_DOCKER=1 go test -v -count=1 -timeout 30m ./internal/installer/harnes
       the previous installation (build/checkout, activation, start, and health
       failure reruns on the Alpine fixture all restore the prior binary, env,
       and running service).
-- [ ] CI uses approved registry mirrors and cleans every test container/network.
+- [x] CI uses approved registry mirrors and cleans every test container/network.
 
 ## Verified Test Evidence (work packages 1-4)
 
@@ -483,3 +569,27 @@ Recorded 2026-08-12 from `make test-ssh` (Docker Engine 29.7.1):
   tests above rather than against a real init. The stub tools simulate unit
   state and command flow, not real systemd/OpenRC unit semantics; a real
   init-system smoke test on a booted host remains a follow-up.
+
+Work package 5 (CI) evidence, recorded 2026-08-13:
+
+- `.github/workflows/test-ssh.yml` runs `scripts/ci/test-ssh.sh` on pushes
+  to `master` and on manual `workflow_dispatch` only (never on
+  `pull_request`, because the fixtures execute the checked-out code inside
+  privileged Docker). Concurrency is scoped per ref
+  (`test-ssh-${{ gitea.ref }}`), the job token is `contents: read`, and
+  the script's loopback port-publishing probe passes on a host Docker
+  daemon while its `EXIT` cleanup leaves zero leftover containers,
+  networks, or fixture image tags.
+- A full local `make test-ssh` run (the exact command the CI job executes)
+  passes the complete harness suite: `TestHarnessFixtures` (alpine/ubuntu/
+  arch), host-key mismatch and stability, teardown, `TestSourceInstallFixtures`
+  (work package 3 staging on all three distros), `TestSourceInstallActivationFixtures`
+  (work package 4 activation on all three), activation failure rollback,
+  `--no-start`, and the dirty-checkout preservation test. Pinned
+  `RSMON_TEST_DOCKER_DNS=1.1.1.1` was used because the local network's
+  default resolvers intermittently time out on `rocketgit.ru`.
+- `TestSourceInstallDirtyCheckoutPreservesStaging` and the rollback test's
+  build-failure step now dirty the tracked tree deterministically (appending
+  a marker line to `Makefile`) instead of `git checkout master~1 -- Makefile`,
+  which depended on the last two commits differing and silently stopped
+  dirtying the tree once a commit did not touch that file.

internal/installer/harness/integration_activation_test.go

@@ -258,8 +258,11 @@ func TestSourceInstallActivationFailureRollback(t *testing.T) {
 
 	// 1. Build failure preserves the prior install.
 	//    A dirty tracked tree makes the rerun's checkout refuse before
-	//    the build, so neither staging nor the service changes.
+	//    the build, so neither staging nor the service changes. Appending
-	if _, err := RunCommand(client, "git -C "+shellQuote(res.Plan.BuildDir)+" checkout -q master~1 -- Makefile"); err != nil {
+	//    a marker is deterministic regardless of the cloned history (a
+	//    `git checkout master~1 -- <file>` would depend on whether that
+	//    commit differs from the resolved one).
+	if _, err := RunCommand(client, "printf '\\n# rsmon-worker dirty-tree marker\\n' >> "+shellQuote(res.Plan.BuildDir)+"/Makefile"); err != nil {
 		t.Fatalf("dirty the working tree: %v", err)
 	}
 	before := installedBinarySHA(t, client, a)

internal/installer/harness/integration_test.go

@@ -466,8 +466,12 @@ func TestSourceInstallDirtyCheckoutPreservesStaging(t *testing.T) {
 	defer client.Close() //nolint:errcheck
 
 	// Dirty a tracked file so the rerun's checkout refuses to proceed.
-	// Makefile differs between master and master~1 (unlike go.mod).
+	// Appending a marker is deterministic regardless of the cloned history
-	if _, err := RunCommand(client, "git -C "+shellQuote(res.Plan.BuildDir)+" checkout -q master~1 -- Makefile"); err != nil {
+	// (a `git checkout master~1 -- <file>` would depend on whether that
+	// commit differs from the resolved one), and an appended line to a
+	// tracked file is always an unstaged modification `git diff --quiet`
+	// catches.
+	if _, err := RunCommand(client, "printf '\\n# rsmon-worker dirty-tree marker\\n' >> "+shellQuote(res.Plan.BuildDir)+"/Makefile"); err != nil {
 		t.Fatalf("dirty the working tree: %v", err)
 	}
 	beforeBinary, err := RunCommand(client, "sha256sum "+shellQuote(res.StageBinary))

scripts/ci/test-ssh.sh

@@ -0,0 +1,78 @@
+#!/usr/bin/env bash
+# CI entrypoint for the source-install Docker/OpenSSH E2E matrix
+# (work package 5, docs/source-installation.md).
+#
+# Runs `make test-ssh` (Alpine/Ubuntu/Arch real OpenSSH fixtures over the
+# public network) inside a Docker-capable CI job. It:
+#   - fails fast with a clear message when Docker is unreachable, because the
+#     harness dials fixture SSH ports published on the daemon's loopback;
+#   - runs the full fixture matrix with make's 60m go-test timeout;
+#   - cleans up every leftover test container/network/fixture-image tag on
+#     success AND failure, so an aborted job never leaks test material on the
+#     runner.
+#
+# It never deletes a shared base image: only rsmon-worker-test-* resources and
+# the per-instance rsmon-worker-test/<fixture>-<suffix>:local tags the harness
+# owns are removed.
+set -euo pipefail
+
+# cleanup removes only resources the harness owns. Runs on every exit path.
+cleanup() {
+    # Containers, then networks (which need no attached containers). Filters
+    # are anchored to the harness's own prefix so a resource that merely
+    # contains the substring is never touched.
+    docker ps -aq --filter "name=^rsmon-worker-test-" 2>/dev/null \
+        | xargs -r docker rm -f >/dev/null 2>&1 || true
+    docker network ls -q --filter "name=^rsmon-worker-test-" 2>/dev/null \
+        | xargs -r docker network rm >/dev/null 2>&1 || true
+    # Per-instance fixture image tags only; never a shared base image.
+    docker images -q --filter "reference=rsmon-worker-test/*" 2>/dev/null \
+        | xargs -r docker image rm -f >/dev/null 2>&1 || true
+}
+trap cleanup EXIT
+
+if ! command -v docker >/dev/null 2>&1; then
+    echo "test-ssh CI: docker CLI not found; this job needs a Docker-capable runner" >&2
+    exit 1
+fi
+
+if ! docker info >/dev/null 2>&1; then
+    echo "test-ssh CI: Docker daemon is not reachable; this job needs a working Docker daemon" >&2
+    exit 1
+fi
+
+echo "test-ssh CI: Docker $(docker version --format '{{.Server.Version}}' 2>/dev/null || echo n/a) ready"
+
+# The harness publishes fixture SSH ports to the daemon's 127.0.0.1 and
+# dials them from the test process, so the job must share the daemon's
+# loopback. Probe that cheaply (no Go, no fixture build) before the heavy
+# matrix so a misconfigured runner fails fast instead of after 60m.
+cid="$(docker run -d --name rsmon-worker-test-preflight \
+    -p 127.0.0.1::22 reg.rsxx.ru/library/alpine:3 sleep 300)"
+port="$(docker port "$cid" 22/tcp 2>/dev/null | sed -n 's#.*:##p' | head -n1)"
+if [ -z "$port" ] || ! (exec 3<>"/dev/tcp/127.0.0.1/${port}") 2>/dev/null; then
+    cat >&2 <<EOF
+test-ssh CI: cannot reach a port published on the Docker daemon loopback.
+
+The harness publishes each fixture's SSH port to the daemon's 127.0.0.1 and
+then dials 127.0.0.1:<published-port> from the test process, so the job and
+the Docker daemon must share a loopback network namespace.
+
+Fix (pick one):
+  1. Run this workflow on a host-mode runner (act_runner job executing on the
+     host with the local Docker daemon), OR
+  2. Give the job container host networking with the Docker socket mounted
+     (e.g. container options: --network host plus /var/run/docker.sock), OR
+  3. Run \`make test-ssh\` directly on a machine with a local Docker daemon.
+
+Diagnostics:
+  docker server: $(docker version --format '{{.Server.Version}}' 2>/dev/null || echo unknown)
+  published port: ${port:-none}
+EOF
+    exit 1
+fi
+exec 3>&- 2>/dev/null || true
+
+echo "test-ssh CI: loopback port publishing verified; running the Alpine/Ubuntu/Arch matrix"
+
+make test-ssh