feat(installer): activate source builds atomically
Все проверки выполнены успешно
CI / test (push) Successful in 4m30s
Docker / Build and publish worker image (push) Successful in 17m26s
Все проверки выполнены успешно
CI / test (push) Successful in 4m30s
Docker / Build and publish worker image (push) Successful in 17m26s
Этот коммит содержится в:
Gleb Tv
390
internal/installer/harness/integration_activation_test.go
Обычный файл
390
internal/installer/harness/integration_activation_test.go
Обычный файл
@@ -0,0 +1,390 @@
|
||||
package harness
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"golang.org/x/crypto/ssh"
|
||||
|
||||
"rocketgit.ru/rsmon/worker/internal/installer"
|
||||
"rocketgit.ru/rsmon/worker/internal/sshinstall"
|
||||
)
|
||||
|
||||
// testWorkerToken is the operator-supplied worker token used by the
|
||||
// activation E2E tests. The fixture has no reachable control plane, but
|
||||
// the worker still binds its webapp and serves /healthz, which is what
|
||||
// activation verifies.
|
||||
const testWorkerToken = "e2e-activation-test-token"
|
||||
|
||||
// activationOptions builds the work-package-4 source-install options for
|
||||
// a started fixture: full activation with a fixed test token, honoring
|
||||
// the RSMON_TEST_SOURCE_REPO/BRANCH overrides the staging tests use.
|
||||
func activationOptions(h *Harness, f Fixture) installer.SourceInstallOptions {
|
||||
opts := installer.SourceInstallOptions{
|
||||
SSHOptions: installer.SSHOptions{
|
||||
Host: "127.0.0.1",
|
||||
Port: h.Port(),
|
||||
User: f.UserOrDefault(),
|
||||
IdentityFile: testKeyPath(),
|
||||
KnownHostsFile: h.KnownHostsPath(),
|
||||
},
|
||||
Activation: installer.ActivationOptions{
|
||||
Activate: true,
|
||||
URL: "https://rsmon.ru",
|
||||
Token: testWorkerToken,
|
||||
},
|
||||
}
|
||||
if repo := strings.TrimSpace(os.Getenv("RSMON_TEST_SOURCE_REPO")); repo != "" {
|
||||
opts.Repo = repo
|
||||
}
|
||||
if branch := strings.TrimSpace(os.Getenv("RSMON_TEST_SOURCE_BRANCH")); branch != "" {
|
||||
opts.Branch = branch
|
||||
}
|
||||
return opts
|
||||
}
|
||||
|
||||
// fixtureByName returns the fixture entry for a fixture name.
|
||||
func fixtureByName(t *testing.T, name string) Fixture {
|
||||
t.Helper()
|
||||
for _, f := range Fixtures() {
|
||||
if f.Name == name {
|
||||
return f
|
||||
}
|
||||
}
|
||||
t.Fatalf("no fixture named %q", name)
|
||||
return Fixture{}
|
||||
}
|
||||
|
||||
// startActivatedFixture starts a fixture container and runs a full
|
||||
// source install with activation against it, returning the harness, the
|
||||
// options (so a test can rerun with modifications), the result, and a
|
||||
// live SSH client. The caller owns client.Close and h.Stop (registered as
|
||||
// a test cleanup).
|
||||
func startActivatedFixture(t *testing.T, name string) (*Harness, installer.SourceInstallOptions, *installer.SourceInstallResult, *ssh.Client) {
|
||||
t.Helper()
|
||||
f := fixtureByName(t, name)
|
||||
h, err := New("activate-"+name, f)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute)
|
||||
defer cancel()
|
||||
if err := h.Start(ctx); err != nil {
|
||||
t.Fatalf("start %s fixture: %v", name, err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := h.Stop(); err != nil {
|
||||
t.Errorf("stop %s fixture: %v", name, err)
|
||||
}
|
||||
})
|
||||
opts := activationOptions(h, f)
|
||||
res, err := installer.SourceInstall(opts)
|
||||
if err != nil {
|
||||
t.Fatalf("activated source install on %s: %v", name, err)
|
||||
}
|
||||
client, err := h.Dial()
|
||||
if err != nil {
|
||||
t.Fatalf("dial %s after install: %v", name, err)
|
||||
}
|
||||
return h, opts, res, client
|
||||
}
|
||||
|
||||
// countWorkerProcesses returns the number of running processes whose
|
||||
// command line is exactly the installed worker binary. The anchored -f
|
||||
// pattern matches both busybox and procps pgrep (busybox's -x matches the
|
||||
// full argv, so -x cannot be used portably).
|
||||
func countWorkerProcesses(t *testing.T, client *ssh.Client, binary string) string {
|
||||
t.Helper()
|
||||
out, err := RunCommand(client, "pgrep -f '^"+binary+"$' | wc -l")
|
||||
if err != nil {
|
||||
t.Fatalf("count worker processes: %v", err)
|
||||
}
|
||||
return strings.TrimSpace(string(out))
|
||||
}
|
||||
|
||||
// assertActivatedService verifies the full work-package-4 remote state:
|
||||
// the installed layout and permissions, the native service definition for
|
||||
// the detected init, exactly one running worker process, the pid file,
|
||||
// and a live /healthz (via the installed binary's own liveness
|
||||
// subcommand).
|
||||
func assertActivatedService(t *testing.T, client *ssh.Client, name string, res *installer.SourceInstallResult) {
|
||||
t.Helper()
|
||||
a := res.Activation
|
||||
if a == nil {
|
||||
t.Fatalf("%s: activation result missing", name)
|
||||
}
|
||||
|
||||
out, err := RunCommand(client, "test -x "+shellQuote(a.Binary)+" && echo BIN_OK")
|
||||
if err != nil || !strings.Contains(string(out), "BIN_OK") {
|
||||
t.Fatalf("%s: installed binary not present/executable at %s: %q, %v", name, a.Binary, out, err)
|
||||
}
|
||||
|
||||
envMode, err := RunCommand(client, "stat -c '%a' "+shellQuote(a.EnvFile))
|
||||
if err != nil || strings.TrimSpace(string(envMode)) != "600" {
|
||||
t.Fatalf("%s: env file mode = %q, want 600 (%v)", name, envMode, err)
|
||||
}
|
||||
cfgMode, err := RunCommand(client, "stat -c '%a' "+shellQuote(a.ConfigDir))
|
||||
if err != nil || strings.TrimSpace(string(cfgMode)) != "750" {
|
||||
t.Fatalf("%s: config dir mode = %q, want 750 (%v)", name, cfgMode, err)
|
||||
}
|
||||
if _, err := RunCommand(client, "test -d "+shellQuote(a.DataDir)+"/webapp && echo DATA_OK"); err != nil {
|
||||
t.Fatalf("%s: data dir not created at %s: %v", name, a.DataDir, err)
|
||||
}
|
||||
|
||||
// The fixtures run no init system (sshd is PID 1), so activation
|
||||
// must use the embedded supervisor; the native service definition is
|
||||
// still installed for real hosts.
|
||||
if a.Supervisor != "none" {
|
||||
t.Fatalf("%s: supervisor = %q, want none in the fixture", name, a.Supervisor)
|
||||
}
|
||||
unitMode := "644"
|
||||
if res.Detection.InitSystem == sshinstall.InitOpenRC {
|
||||
unitMode = "755"
|
||||
}
|
||||
if a.UnitFile == "" {
|
||||
t.Fatalf("%s: no service definition installed", name)
|
||||
}
|
||||
if _, err := RunCommand(client, "test -f "+shellQuote(a.UnitFile)+" && echo UNIT_OK"); err != nil {
|
||||
t.Fatalf("%s: unit file missing at %s: %v", name, a.UnitFile, err)
|
||||
}
|
||||
gotUnitMode, err := RunCommand(client, "stat -c '%a' "+shellQuote(a.UnitFile))
|
||||
if err != nil || strings.TrimSpace(string(gotUnitMode)) != unitMode {
|
||||
t.Fatalf("%s: unit mode = %q, want %s (%v)", name, gotUnitMode, unitMode, err)
|
||||
}
|
||||
|
||||
// Exactly one worker process and a live /healthz.
|
||||
if got := countWorkerProcesses(t, client, a.Binary); got != "1" {
|
||||
t.Fatalf("%s: worker process count = %q, want 1", name, got)
|
||||
}
|
||||
if out, err := RunCommand(client, shellQuote(a.Binary)+" liveness"); err != nil || !strings.Contains(string(out), "ok") {
|
||||
t.Fatalf("%s: installed worker /healthz check failed: %q, %v", name, out, err)
|
||||
}
|
||||
pidOut, err := RunCommand(client, "cat "+shellQuote(a.DataDir)+"/worker.pid")
|
||||
if err != nil {
|
||||
t.Fatalf("%s: pid file missing: %v", name, err)
|
||||
}
|
||||
if _, err := RunCommand(client, "kill -0 "+strings.TrimSpace(string(pidOut))+" && echo PID_OK"); err != nil {
|
||||
t.Fatalf("%s: pid %s not alive: %v", name, pidOut, err)
|
||||
}
|
||||
}
|
||||
|
||||
// assertNoActivationLeaks verifies a rerun left no backup dir, no lock,
|
||||
// no /tmp upload leftovers, and no extra worker processes.
|
||||
func assertNoActivationLeaks(t *testing.T, client *ssh.Client, name string, dataDir string) {
|
||||
t.Helper()
|
||||
out, err := RunCommand(client,
|
||||
"leak=0; [ -e "+shellQuote(dataDir)+"/.rsmon-backup ] && leak=1; "+
|
||||
"[ -d "+shellQuote(dataDir)+"/.rsmon-activate.lock ] && leak=1; "+
|
||||
"ls /tmp 2>/dev/null | grep -q '^rsmon-worker-act\\.' && leak=1; "+
|
||||
"[ \"$leak\" -eq 1 ] && echo LEAK || echo CLEAN")
|
||||
if err != nil || strings.TrimSpace(string(out)) != "CLEAN" {
|
||||
t.Fatalf("%s: activation leftovers after rerun: %q, %v", name, out, err)
|
||||
}
|
||||
}
|
||||
|
||||
// installedBinarySHA captures the SHA-256 of the installed worker binary
|
||||
// so failure tests can prove the prior binary is preserved.
|
||||
func installedBinarySHA(t *testing.T, client *ssh.Client, a *installer.ActivationResult) []byte {
|
||||
t.Helper()
|
||||
out, err := RunCommand(client, "sha256sum "+shellQuote(a.Binary))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return bytes.TrimSpace(out)
|
||||
}
|
||||
|
||||
// TestSourceInstallActivationFixtures is the work-package-4 success and
|
||||
// idempotency acceptance test: on every fixture a full source install
|
||||
// ends with an atomically installed binary, a mode-0600 env file, the
|
||||
// data dir, a service definition for the detected init, and a running
|
||||
// verified worker. A rerun succeeds, keeps a single worker process,
|
||||
// rewrites the env deterministically, and leaks no temp or backup files.
|
||||
//
|
||||
// Opt-in: set RSMON_TEST_DOCKER=1 (see make test-ssh).
|
||||
func TestSourceInstallActivationFixtures(t *testing.T) {
|
||||
SkipUnlessEnabled(t)
|
||||
|
||||
for _, f := range Fixtures() {
|
||||
f := f
|
||||
t.Run(f.Name, func(t *testing.T) {
|
||||
_, opts, res, client := startActivatedFixture(t, f.Name)
|
||||
defer client.Close() //nolint:errcheck
|
||||
assertActivatedService(t, client, f.Name, res)
|
||||
|
||||
// Rerun: idempotent, single service, no leaks.
|
||||
res2, err := installer.SourceInstall(opts)
|
||||
if err != nil {
|
||||
t.Fatalf("%s: activated rerun: %v", f.Name, err)
|
||||
}
|
||||
if res2.Activation == nil || res2.ResolvedCommit != res.ResolvedCommit {
|
||||
t.Fatalf("%s: rerun lost activation/commit: %+v", f.Name, res2)
|
||||
}
|
||||
assertActivatedService(t, client, f.Name, res2)
|
||||
assertNoActivationLeaks(t, client, f.Name, res2.Activation.DataDir)
|
||||
|
||||
envAfter, err := RunCommand(client, "cat "+shellQuote(res2.Activation.EnvFile))
|
||||
if err != nil {
|
||||
t.Fatalf("%s: read env after rerun: %v", f.Name, err)
|
||||
}
|
||||
if !strings.Contains(string(envAfter), "RSMON_TOKEN="+testWorkerToken) {
|
||||
t.Fatalf("%s: env lost the worker token after rerun: %q", f.Name, envAfter)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestSourceInstallActivationFailureRollback is the work-package-4
|
||||
// rollback acceptance test. After a successful activated install on the
|
||||
// Alpine fixture, each rerun is forced to fail at a different stage -
|
||||
// build (dirty checkout), activation (atomic swap sabotaged), start (the
|
||||
// worker process dies at boot), and health (/healthz unreachable) - and
|
||||
// every failure must leave the previous working install running with the
|
||||
// same binary, the same single process, and /healthz answering.
|
||||
//
|
||||
// Opt-in: set RSMON_TEST_DOCKER=1 (see make test-ssh).
|
||||
func TestSourceInstallActivationFailureRollback(t *testing.T) {
|
||||
SkipUnlessEnabled(t)
|
||||
|
||||
const fixture = "alpine"
|
||||
_, opts, res, client := startActivatedFixture(t, fixture)
|
||||
defer client.Close() //nolint:errcheck
|
||||
a := res.Activation
|
||||
assertActivatedService(t, client, fixture, res)
|
||||
|
||||
// 1. Build failure preserves the prior install.
|
||||
// A dirty tracked tree makes the rerun's checkout refuse before
|
||||
// the build, so neither staging nor the service changes.
|
||||
if _, err := RunCommand(client, "git -C "+shellQuote(res.Plan.BuildDir)+" checkout -q master~1 -- Makefile"); err != nil {
|
||||
t.Fatalf("dirty the working tree: %v", err)
|
||||
}
|
||||
before := installedBinarySHA(t, client, a)
|
||||
_, err := installer.SourceInstall(opts)
|
||||
if err == nil || !strings.Contains(err.Error(), "check out branch") {
|
||||
t.Fatalf("rerun on dirty tree err = %v, want checkout failure", err)
|
||||
}
|
||||
assertActivatedService(t, client, fixture, res)
|
||||
if !bytes.Equal(before, installedBinarySHA(t, client, a)) {
|
||||
t.Fatalf("installed binary changed after build-failure rerun")
|
||||
}
|
||||
// Restore the tree so later reruns can check out again.
|
||||
if _, err := RunCommand(client, "git -C "+shellQuote(res.Plan.BuildDir)+" checkout -q HEAD -- Makefile"); err != nil {
|
||||
t.Fatalf("restore working tree: %v", err)
|
||||
}
|
||||
|
||||
// 2. Activation failure preserves the prior install.
|
||||
// Pre-creating the atomic-swap temp path as a directory makes the
|
||||
// binary install step fail mid-activation; the rollback trap must
|
||||
// restore the previous binary/env and restart the prior service.
|
||||
if _, err := RunCommand(client, "mkdir -p /usr/local/bin/rsmon-worker.new"); err != nil {
|
||||
t.Fatalf("sabotage the atomic binary swap: %v", err)
|
||||
}
|
||||
_, err = installer.SourceInstall(opts)
|
||||
if err == nil || !strings.Contains(err.Error(), "activate worker service") {
|
||||
t.Fatalf("activation-failure rerun err = %v, want activate failure", err)
|
||||
}
|
||||
assertActivatedService(t, client, fixture, res)
|
||||
if !bytes.Equal(before, installedBinarySHA(t, client, a)) {
|
||||
t.Fatalf("installed binary changed after activation-failure rerun")
|
||||
}
|
||||
assertNoActivationLeaks(t, client, fixture, a.DataDir)
|
||||
if _, err := RunCommand(client, "rm -rf /usr/local/bin/rsmon-worker.new"); err != nil {
|
||||
t.Fatalf("clean the swap sabotage: %v", err)
|
||||
}
|
||||
|
||||
// 3. Start failure preserves the prior install.
|
||||
// WORKER_CLUSTER_ENABLED=true without WORKER_LOGIN/PASSWORD makes
|
||||
// the new worker exit at boot (cluster init is fatal), so the
|
||||
// activation supervisor detects the process died and rolls back.
|
||||
envPath := filepath.Join(t.TempDir(), "worker.env")
|
||||
startEnv := "RSMON_URL=https://rsmon.ru\nRSMON_TOKEN=" + testWorkerToken + "\nWORKER_CLUSTER_ENABLED=true\n"
|
||||
if err := os.WriteFile(envPath, []byte(startEnv), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
startOpts := opts
|
||||
startOpts.Activation.EnvFile = envPath
|
||||
_, err = installer.SourceInstall(startOpts)
|
||||
if err == nil || !strings.Contains(err.Error(), "start failure") {
|
||||
t.Fatalf("start-failure rerun err = %v, want start failure", err)
|
||||
}
|
||||
assertActivatedService(t, client, fixture, res)
|
||||
if !bytes.Equal(before, installedBinarySHA(t, client, a)) {
|
||||
t.Fatalf("installed binary changed after start-failure rerun")
|
||||
}
|
||||
assertNoActivationLeaks(t, client, fixture, a.DataDir)
|
||||
|
||||
// 4. Health failure preserves the prior install.
|
||||
// WORKER_HOST=255.255.255.255 makes the new worker bind fail and
|
||||
// /healthz unreachable; the process stays up but the health gate
|
||||
// fails and the rollback restores the prior loopback deployment.
|
||||
healthOpts := opts
|
||||
healthOpts.Activation.Host = "255.255.255.255"
|
||||
_, err = installer.SourceInstall(healthOpts)
|
||||
if err == nil || !strings.Contains(err.Error(), "health failure") {
|
||||
t.Fatalf("health-failure rerun err = %v, want health failure", err)
|
||||
}
|
||||
assertActivatedService(t, client, fixture, res)
|
||||
if !bytes.Equal(before, installedBinarySHA(t, client, a)) {
|
||||
t.Fatalf("installed binary changed after health-failure rerun")
|
||||
}
|
||||
assertNoActivationLeaks(t, client, fixture, a.DataDir)
|
||||
}
|
||||
|
||||
// TestSourceInstallActivationNoStart installs the full layout (binary,
|
||||
// env, data dir, service definition) without starting the worker, and
|
||||
// asserts nothing runs afterward. Reruns stay idempotent.
|
||||
//
|
||||
// Opt-in: set RSMON_TEST_DOCKER=1 (see make test-ssh).
|
||||
func TestSourceInstallActivationNoStart(t *testing.T) {
|
||||
SkipUnlessEnabled(t)
|
||||
|
||||
const fixture = "alpine"
|
||||
f := fixtureByName(t, fixture)
|
||||
h, err := New("nostart-"+fixture, f)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute)
|
||||
defer cancel()
|
||||
if err := h.Start(ctx); err != nil {
|
||||
t.Fatalf("start %s fixture: %v", fixture, err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := h.Stop(); err != nil {
|
||||
t.Errorf("stop: %v", err)
|
||||
}
|
||||
})
|
||||
opts := activationOptions(h, f)
|
||||
opts.Activation.NoStart = true
|
||||
res, err := installer.SourceInstall(opts)
|
||||
if err != nil {
|
||||
t.Fatalf("no-start install: %v", err)
|
||||
}
|
||||
client, err := h.Dial()
|
||||
if err != nil {
|
||||
t.Fatalf("dial: %v", err)
|
||||
}
|
||||
defer client.Close() //nolint:errcheck
|
||||
a := res.Activation
|
||||
if a == nil || a.Started {
|
||||
t.Fatalf("no-start activation result = %+v", a)
|
||||
}
|
||||
if out, err := RunCommand(client, "test -x "+shellQuote(a.Binary)+" && test -f "+shellQuote(a.EnvFile)+" && echo INSTALLED"); err != nil || !strings.Contains(string(out), "INSTALLED") {
|
||||
t.Fatalf("no-start did not install the layout: %q, %v", out, err)
|
||||
}
|
||||
if got := countWorkerProcesses(t, client, a.Binary); got != "0" {
|
||||
t.Fatalf("no-start left a running worker: %q", got)
|
||||
}
|
||||
// A rerun is idempotent and still starts nothing.
|
||||
if _, err := installer.SourceInstall(opts); err != nil {
|
||||
t.Fatalf("no-start rerun: %v", err)
|
||||
}
|
||||
if got := countWorkerProcesses(t, client, a.Binary); got != "0" {
|
||||
t.Fatalf("no-start rerun left a running worker: %q", got)
|
||||
}
|
||||
assertNoActivationLeaks(t, client, fixture, a.DataDir)
|
||||
}
|
||||
@@ -325,6 +325,9 @@ func TestSourceInstallFixtures(t *testing.T) {
|
||||
IdentityFile: testKeyPath(),
|
||||
KnownHostsFile: h.KnownHostsPath(),
|
||||
},
|
||||
// Staging-only: service activation (work package 4) is
|
||||
// exercised by the dedicated activation tests.
|
||||
Activation: installer.ActivationOptions{Activate: false},
|
||||
}
|
||||
if repo := strings.TrimSpace(os.Getenv("RSMON_TEST_SOURCE_REPO")); repo != "" {
|
||||
opts.Repo = repo
|
||||
@@ -446,6 +449,7 @@ func TestSourceInstallDirtyCheckoutPreservesStaging(t *testing.T) {
|
||||
IdentityFile: testKeyPath(),
|
||||
KnownHostsFile: h.KnownHostsPath(),
|
||||
},
|
||||
Activation: installer.ActivationOptions{Activate: false},
|
||||
}
|
||||
if repo := strings.TrimSpace(os.Getenv("RSMON_TEST_SOURCE_REPO")); repo != "" {
|
||||
opts.Repo = repo
|
||||
|
||||
@@ -478,6 +478,16 @@ func ValidateEnvironmentFile(path string) error {
|
||||
if err != nil {
|
||||
return fmt.Errorf("read worker environment: %w", err)
|
||||
}
|
||||
_, err = parseEnvironmentContent(data)
|
||||
return err
|
||||
}
|
||||
|
||||
// parseEnvironmentContent validates systemd-safe KEY=VALUE environment
|
||||
// content and returns the parsed map. Values must not use quoting,
|
||||
// interpolation, or whitespace; RSMON_URL and RSMON_TOKEN are required and
|
||||
// must not be duplicated. Operating on the already-read bytes (rather
|
||||
// than a path) keeps callers free of read-to-validate TOCTOU races.
|
||||
func parseEnvironmentContent(data []byte) (map[string]string, error) {
|
||||
values := make(map[string]string)
|
||||
seenRequired := make(map[string]bool)
|
||||
for number, line := range strings.Split(string(data), "\n") {
|
||||
@@ -486,30 +496,30 @@ func ValidateEnvironmentFile(path string) error {
|
||||
continue
|
||||
}
|
||||
if strings.ContainsRune(line, '\r') {
|
||||
return fmt.Errorf("worker environment line %d contains a carriage return", lineNumber)
|
||||
return nil, fmt.Errorf("worker environment line %d contains a carriage return", lineNumber)
|
||||
}
|
||||
key, value, ok := strings.Cut(line, "=")
|
||||
if !ok || !envKeyPattern.MatchString(key) {
|
||||
return fmt.Errorf("worker environment line %d must use KEY=VALUE syntax", lineNumber)
|
||||
return nil, fmt.Errorf("worker environment line %d must use KEY=VALUE syntax", lineNumber)
|
||||
}
|
||||
if err := validateEnvValue(key, value); err != nil {
|
||||
return fmt.Errorf("worker environment line %d: %w", lineNumber, err)
|
||||
return nil, fmt.Errorf("worker environment line %d: %w", lineNumber, err)
|
||||
}
|
||||
if key == "RSMON_URL" || key == "RSMON_TOKEN" {
|
||||
if seenRequired[key] {
|
||||
return fmt.Errorf("worker environment line %d duplicates %s", lineNumber, key)
|
||||
return nil, fmt.Errorf("worker environment line %d duplicates %s", lineNumber, key)
|
||||
}
|
||||
seenRequired[key] = true
|
||||
}
|
||||
values[key] = value
|
||||
}
|
||||
if err := ValidateURL(values["RSMON_URL"]); err != nil {
|
||||
return err
|
||||
return nil, err
|
||||
}
|
||||
if err := ValidateToken(values["RSMON_TOKEN"]); err != nil {
|
||||
return err
|
||||
return nil, err
|
||||
}
|
||||
return nil
|
||||
return values, nil
|
||||
}
|
||||
|
||||
// validateEnvValue enforces the value rules shared by the strict env
|
||||
|
||||
769
internal/installer/sourceactivate.go
Обычный файл
769
internal/installer/sourceactivate.go
Обычный файл
@@ -0,0 +1,769 @@
|
||||
package installer
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"rocketgit.ru/rsmon/worker/internal/sshinstall"
|
||||
)
|
||||
|
||||
// ActivationOptions drives work package 4 of docs/source-installation.md:
|
||||
// the atomic activation of the staged worker binary. It reuses the
|
||||
// classic installer's environment resolution (resolveInstallEnv /
|
||||
// renderEnvFile), on-disk layout (resolvePaths), and hardened systemd
|
||||
// unit renderer, then installs the binary, env file, data dir, and the
|
||||
// detected init's service definition atomically, starts/restarts the
|
||||
// worker, and verifies the process and /healthz before declaring the
|
||||
// activation a success. Any activation/start/health failure rolls back to
|
||||
// the previous working install.
|
||||
type ActivationOptions struct {
|
||||
// Activate performs the atomic install + service activation after the
|
||||
// staging build. When false SourceInstall stops at the staging build
|
||||
// (the work-package-3 boundary) and touches no service configuration.
|
||||
Activate bool
|
||||
|
||||
// Name is the instance name ("" = the primary rsmon-worker service).
|
||||
// A named instance gets rsmon-worker-<name> paths and its own unit,
|
||||
// matching the classic installer's multi-instance layout.
|
||||
Name string
|
||||
// URL is RSMON_URL (default https://rsmon.ru).
|
||||
URL string
|
||||
// Token is the worker API token (RSMON_TOKEN). Required for
|
||||
// activation; prefer supplying it through a file at the CLI layer so
|
||||
// it never appears in the process list.
|
||||
Token string
|
||||
// PublicURL is the advertised public origin (PUBLIC_URL).
|
||||
PublicURL string
|
||||
// Host is the worker webapp bind address (WORKER_HOST; default
|
||||
// 127.0.0.1 as resolved by the classic installer).
|
||||
Host string
|
||||
// Port is the worker webapp bind port (WORKER_PORT; default 27401,
|
||||
// required for named instances).
|
||||
Port string
|
||||
// Login / Password are the operator-console basic-auth credentials
|
||||
// (WORKER_LOGIN / WORKER_PASSWORD). Both must be set or both empty.
|
||||
Login string
|
||||
Password string
|
||||
// EnvFile is a local systemd-safe env file uploaded instead of the
|
||||
// individual knobs. Validated with the classic strict parser.
|
||||
EnvFile string
|
||||
// NoStart installs the binary, env, data dir, and service definition
|
||||
// without starting or restarting the worker.
|
||||
NoStart bool
|
||||
}
|
||||
|
||||
// ActivationResult records where and how the staged worker was activated.
|
||||
type ActivationResult struct {
|
||||
Binary string // installed binary path
|
||||
ConfigDir string
|
||||
EnvFile string
|
||||
DataDir string
|
||||
UnitFile string // empty when no service definition was installed
|
||||
UnitName string // systemd unit / rc-service name
|
||||
Supervisor string // "systemd", "openrc", or "supervisor" (embedded fallback)
|
||||
Started bool
|
||||
}
|
||||
|
||||
// installOptions projects the activation knobs onto the classic
|
||||
// installer's option shape so the shared env resolution is reused.
|
||||
func (a ActivationOptions) installOptions() InstallOptions {
|
||||
return InstallOptions{
|
||||
URL: a.URL,
|
||||
Token: a.Token,
|
||||
PublicURL: a.PublicURL,
|
||||
Host: a.Host,
|
||||
Port: a.Port,
|
||||
Login: a.Login,
|
||||
Password: a.Password,
|
||||
Name: strings.TrimSpace(a.Name),
|
||||
NoStart: a.NoStart,
|
||||
EnvFile: a.EnvFile,
|
||||
}
|
||||
}
|
||||
|
||||
// renderEnv resolves the full worker environment exactly as the classic
|
||||
// installer would and renders the canonical systemd-safe env file. The
|
||||
// env file (when configured) is read exactly once and validated from the
|
||||
// in-memory bytes, so a hostile local writer cannot swap the file
|
||||
// between the read used for validation and the read used for the render.
|
||||
func (a ActivationOptions) renderEnv() ([]byte, error) {
|
||||
name := strings.TrimSpace(a.Name)
|
||||
if err := validateInstanceName(name); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var fileEnv map[string]string
|
||||
if a.EnvFile != "" {
|
||||
data, err := os.ReadFile(a.EnvFile)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("--env-file: %w", err)
|
||||
}
|
||||
fe, perr := parseEnvironmentContent(data)
|
||||
if perr != nil {
|
||||
return nil, fmt.Errorf("--env-file: %w", perr)
|
||||
}
|
||||
fileEnv = fe
|
||||
}
|
||||
values, err := resolveInstallEnv(a.installOptions(), name, fileEnv)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return renderEnvFile(values), nil
|
||||
}
|
||||
|
||||
// unitContent renders the service definition for the detected init
|
||||
// system. systemd gets the classic hardened unit; OpenRC gets a native
|
||||
// init script. An unknown init produces no unit (the no-service gate:
|
||||
// the embedded supervisor still installs and manages the worker, but no
|
||||
// native service definition is written).
|
||||
func unitContent(init sshinstall.InitSystem, p paths) (content string, file string, mode string) {
|
||||
svcName := strings.TrimSuffix(p.unitName, ".service")
|
||||
switch init {
|
||||
case sshinstall.InitSystemd:
|
||||
return systemdUnitFor(p), p.unitFile, "0644"
|
||||
case sshinstall.InitOpenRC:
|
||||
return openrcInitFor(p), "/etc/init.d/" + svcName, "0755"
|
||||
default:
|
||||
return "", "", ""
|
||||
}
|
||||
}
|
||||
|
||||
// runningInitScript reports the init system that is *actually running*
|
||||
// (not merely installed): systemd, openrc, or none. The installer writes
|
||||
// the service definition for the detected init but drives start/restart
|
||||
// through whichever supervisor is usable right now; a container or chroot
|
||||
// where no init runs falls back to the embedded supervisor.
|
||||
const runningInitScript = `set -eu
|
||||
if [ -d /run/systemd/system ] && command -v systemctl >/dev/null 2>&1; then
|
||||
echo systemd
|
||||
elif [ -e /run/openrc/softlevel ] && command -v rc-service >/dev/null 2>&1; then
|
||||
echo openrc
|
||||
else
|
||||
echo none
|
||||
fi`
|
||||
|
||||
// normalizeSupervisor bounds the running-init probe output to the three
|
||||
// supervisor kinds the activation script understands.
|
||||
func normalizeSupervisor(raw string) string {
|
||||
switch strings.TrimSpace(raw) {
|
||||
case "systemd", "openrc":
|
||||
return strings.TrimSpace(raw)
|
||||
default:
|
||||
return "none"
|
||||
}
|
||||
}
|
||||
|
||||
// activateScript renders the single idempotent remote activation script.
|
||||
// It is structured as a transaction over the previous install:
|
||||
//
|
||||
// 1. acquire an activation lock so concurrent runs cannot mutate the same
|
||||
// deployment, and recover any interrupted prior activation from its
|
||||
// leftover backup marker;
|
||||
// 2. validate the staged binary (fails before any state is touched);
|
||||
// 3. snapshot the prior binary/env/unit (and the unit's enable state)
|
||||
// into a backup dir and write a recovery marker;
|
||||
// 4. arm a rollback EXIT/HUP/INT/TERM trap that restores the snapshot,
|
||||
// restores the prior enable state (or disables a freshly-installed
|
||||
// unit on a fresh failure), and restarts the prior service;
|
||||
// 5. install the new binary, env (mode 0600), data dir, and service
|
||||
// definition atomically (temp file + rename);
|
||||
// 6. start/restart through the active supervisor and verify the service
|
||||
// stays active (systemd `is-active` / rc-service `status` / the
|
||||
// embedded supervisor's zombie-aware pid check) and answers /healthz;
|
||||
// 7. on success, drop the backup and release the lock; on failure the
|
||||
// trap restores the prior install and the script exits non-zero.
|
||||
//
|
||||
// Reruns are idempotent: restart always stops the previous instance
|
||||
// first, so exactly one worker process exists, and every temp file
|
||||
// (backup dir, .new files, uploaded env/unit, lock) is removed on success
|
||||
// and failure.
|
||||
func activateScript(p activateParams) string {
|
||||
r := strings.NewReplacer(
|
||||
"@@STAGE@@", shellQuote(p.Stage),
|
||||
"@@BINARY@@", shellQuote(p.Binary),
|
||||
"@@CONFIG_DIR@@", shellQuote(p.ConfigDir),
|
||||
"@@ENV_FILE@@", shellQuote(p.EnvFile),
|
||||
"@@ENV_TMP@@", shellQuote(p.EnvTmp),
|
||||
"@@DATA_DIR@@", shellQuote(p.DataDir),
|
||||
"@@UNIT_TMP@@", shellQuote(p.UnitTmp),
|
||||
"@@UNIT_FILE@@", shellQuote(p.UnitFile),
|
||||
"@@UNIT_MODE@@", p.UnitMode,
|
||||
"@@UNIT_NAME@@", shellQuote(p.UnitName),
|
||||
"@@RC_NAME@@", shellQuote(p.RCName),
|
||||
"@@SUPERVISOR@@", p.Supervisor,
|
||||
"@@NO_START@@", p.NoStart,
|
||||
)
|
||||
return r.Replace(activateTemplate)
|
||||
}
|
||||
|
||||
// activateTemplate is the remote script body. Sentinels (@@..@@) are
|
||||
// substituted by activateScript; every operator-controlled value is
|
||||
// single-quoted and validated on the controller side first.
|
||||
const activateTemplate = `set -eu
|
||||
|
||||
stage=@@STAGE@@
|
||||
binary=@@BINARY@@
|
||||
config_dir=@@CONFIG_DIR@@
|
||||
env_file=@@ENV_FILE@@
|
||||
env_tmp=@@ENV_TMP@@
|
||||
data_dir=@@DATA_DIR@@
|
||||
unit_tmp=@@UNIT_TMP@@
|
||||
unit_file=@@UNIT_FILE@@
|
||||
unit_mode=@@UNIT_MODE@@
|
||||
unit_name=@@UNIT_NAME@@
|
||||
rc_name=@@RC_NAME@@
|
||||
supervisor=@@SUPERVISOR@@
|
||||
no_start=@@NO_START@@
|
||||
|
||||
backup_dir="$data_dir/.rsmon-backup"
|
||||
lock_dir="$data_dir/.rsmon-activate.lock"
|
||||
pid_file="$data_dir/worker.pid"
|
||||
log_file="$data_dir/worker.log"
|
||||
|
||||
run_env() {
|
||||
# Execute "$@" with a fresh environment built only from the env file
|
||||
# plus PATH, HOME, and the webapp data dir. Sourcing the env file in a
|
||||
# clean env -i shell prevents variables from a previous activation
|
||||
# (for example WORKER_CLUSTER_ENABLED) from leaking into the new
|
||||
# worker process after a rollback restores an older env file.
|
||||
env -i PATH="/usr/bin:/bin:/sbin:/usr/sbin" HOME="$data_dir" \
|
||||
RSMON_WEBAPP_DATA_DIR="$data_dir/webapp" \
|
||||
RSMON_WORKER_ENV_FILE="$env_file" sh -c '
|
||||
set -a
|
||||
. "$RSMON_WORKER_ENV_FILE"
|
||||
set +a
|
||||
unset RSMON_WORKER_ENV_FILE
|
||||
exec "$@"
|
||||
' sh "$@"
|
||||
}
|
||||
|
||||
process_up() {
|
||||
# True when the given pid is a live, non-zombie process. A zombie
|
||||
# still answers kill -0 (its task struct exists until reaped), and in
|
||||
# a container where PID 1 does not reap children a killed worker can
|
||||
# linger as a zombie for a long time, so the /proc state must be
|
||||
# checked explicitly.
|
||||
[ -n "$1" ] || return 1
|
||||
kill -0 "$1" 2>/dev/null || return 1
|
||||
state="$(awk '{print $3}' "/proc/$1/stat" 2>/dev/null || true)"
|
||||
[ "$state" != "Z" ] || return 1
|
||||
}
|
||||
|
||||
process_is_worker() {
|
||||
# True when the given pid's executable is (or was, before an atomic
|
||||
# binary swap) the configured worker binary, so a stale or recycled
|
||||
# pid file can never make the supervisor kill an unrelated process.
|
||||
[ -n "$1" ] || return 1
|
||||
exe="$(readlink "/proc/$1/exe" 2>/dev/null || true)"
|
||||
case "$exe" in
|
||||
"$binary"|"$binary (deleted)") return 0 ;;
|
||||
esac
|
||||
return 1
|
||||
}
|
||||
|
||||
stop_process() {
|
||||
if [ -f "$pid_file" ]; then
|
||||
pid="$(cat "$pid_file" 2>/dev/null || true)"
|
||||
if [ -n "$pid" ] && process_is_worker "$pid" && process_up "$pid"; then
|
||||
kill "$pid" 2>/dev/null || true
|
||||
i=0
|
||||
while [ "$i" -lt 10 ]; do
|
||||
if ! process_up "$pid"; then
|
||||
break
|
||||
fi
|
||||
sleep 1
|
||||
i=$((i + 1))
|
||||
done
|
||||
kill -9 "$pid" 2>/dev/null || true
|
||||
fi
|
||||
fi
|
||||
rm -f "$pid_file"
|
||||
}
|
||||
|
||||
start_process() {
|
||||
stop_process
|
||||
mkdir -p "$data_dir"
|
||||
run_env sh -c 'nohup "$1" >>"$2" 2>&1 & echo $! > "$3"' sh "$binary" "$log_file" "$pid_file"
|
||||
}
|
||||
|
||||
process_alive() {
|
||||
[ -f "$pid_file" ] || return 1
|
||||
pid="$(cat "$pid_file" 2>/dev/null || true)"
|
||||
process_is_worker "$pid" || return 1
|
||||
process_up "$pid"
|
||||
}
|
||||
|
||||
worker_come_up() {
|
||||
# Wait up to 10s for the freshly-started worker to become a live,
|
||||
# non-zombie process running the configured binary. The recorded pid
|
||||
# starts as the nohup/sh child before it execs the worker, so early
|
||||
# readlink checks can transiently see the interpreter; this retry both
|
||||
# tolerates that exec window and catches genuine immediate deaths.
|
||||
i=0
|
||||
while [ "$i" -lt 10 ]; do
|
||||
if process_alive; then
|
||||
return 0
|
||||
fi
|
||||
sleep 1
|
||||
i=$((i + 1))
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
health_ok() {
|
||||
run_env "$binary" liveness >/dev/null 2>&1
|
||||
}
|
||||
|
||||
svc_active() {
|
||||
# The health loop verifies the service is active through the
|
||||
# supervisor (never through a pid file for the init-managed paths).
|
||||
case "$supervisor" in
|
||||
systemd) systemctl is-active --quiet "$unit_name" ;;
|
||||
openrc) rc-service "$rc_name" status >/dev/null 2>&1 ;;
|
||||
none) process_alive ;;
|
||||
esac
|
||||
}
|
||||
|
||||
stop_svc() {
|
||||
case "$supervisor" in
|
||||
systemd) systemctl stop "$unit_name" >/dev/null 2>&1 || true ;;
|
||||
openrc) rc-service "$rc_name" stop >/dev/null 2>&1 || true ;;
|
||||
none) stop_process ;;
|
||||
esac
|
||||
}
|
||||
|
||||
restart_svc() {
|
||||
case "$supervisor" in
|
||||
systemd)
|
||||
systemctl daemon-reload >/dev/null 2>&1
|
||||
if ! systemctl restart "$unit_name" >/dev/null 2>&1; then
|
||||
echo "start failure: systemctl restart $unit_name failed" >&2
|
||||
return 1
|
||||
fi
|
||||
if ! systemctl is-active --quiet "$unit_name"; then
|
||||
echo "start failure: unit $unit_name is not active" >&2
|
||||
return 1
|
||||
fi
|
||||
;;
|
||||
openrc)
|
||||
if ! rc-service "$rc_name" restart >/dev/null 2>&1; then
|
||||
echo "start failure: rc-service restart $rc_name failed" >&2
|
||||
return 1
|
||||
fi
|
||||
if ! rc-service "$rc_name" status >/dev/null 2>&1; then
|
||||
echo "start failure: service $rc_name is not running" >&2
|
||||
return 1
|
||||
fi
|
||||
;;
|
||||
none)
|
||||
start_process
|
||||
if ! worker_come_up; then
|
||||
echo "start failure: worker process did not stay alive" >&2
|
||||
return 1
|
||||
fi
|
||||
;;
|
||||
esac
|
||||
i=0
|
||||
while [ "$i" -lt 30 ]; do
|
||||
if health_ok; then
|
||||
return 0
|
||||
fi
|
||||
# A worker that stops while the health gate is still probing is a
|
||||
# start failure (it never became a stable service), not merely an
|
||||
# unhealthy-but-running one.
|
||||
if ! svc_active; then
|
||||
echo "start failure: worker service is not active" >&2
|
||||
return 1
|
||||
fi
|
||||
sleep 1
|
||||
i=$((i + 1))
|
||||
done
|
||||
echo "health failure: worker did not answer /healthz" >&2
|
||||
return 1
|
||||
}
|
||||
|
||||
# --- unit enable helpers -------------------------------------------------
|
||||
# unit_was_enabled records whether the unit referenced by the current
|
||||
# params is enabled (best-effort; a missing init leaves it disabled).
|
||||
unit_was_enabled() {
|
||||
case "$supervisor" in
|
||||
systemd) systemctl is-enabled "$unit_name" >/dev/null 2>&1 && unit_enabled=1 ;;
|
||||
openrc) [ -e "/etc/runlevels/default/$rc_name" ] && unit_enabled=1 ;;
|
||||
esac
|
||||
return 0
|
||||
}
|
||||
# set_unit_enabled "$1" enables (1) or disables (0) the unit.
|
||||
set_unit_enabled() {
|
||||
case "$supervisor" in
|
||||
systemd)
|
||||
systemctl daemon-reload >/dev/null 2>&1 || true
|
||||
if [ "$1" -eq 1 ]; then
|
||||
systemctl enable "$unit_name" >/dev/null 2>&1 || true
|
||||
else
|
||||
systemctl disable "$unit_name" >/dev/null 2>&1 || true
|
||||
fi
|
||||
;;
|
||||
openrc)
|
||||
if [ "$1" -eq 1 ]; then
|
||||
rc-update add "$rc_name" default >/dev/null 2>&1 || true
|
||||
else
|
||||
rc-update del "$rc_name" default >/dev/null 2>&1 || true
|
||||
fi
|
||||
;;
|
||||
esac
|
||||
return 0
|
||||
}
|
||||
|
||||
# --- activation lock ------------------------------------------------------
|
||||
# Prevent concurrent activations on the same host from racing the
|
||||
# deployed-state mutation. A stale lock (left over from a SIGKILL'd run
|
||||
# whose pid is no longer alive) is broken automatically.
|
||||
acquire_lock() {
|
||||
mkdir -p "$data_dir"
|
||||
if [ -d "$lock_dir" ]; then
|
||||
lockpid="$(cat "$lock_dir/pid" 2>/dev/null || true)"
|
||||
if [ -n "$lockpid" ] && ! kill -0 "$lockpid" 2>/dev/null; then
|
||||
rm -rf "$lock_dir" 2>/dev/null || true
|
||||
fi
|
||||
fi
|
||||
i=0
|
||||
while [ "$i" -lt 60 ]; do
|
||||
if mkdir "$lock_dir" 2>/dev/null; then
|
||||
chmod 0700 "$lock_dir" 2>/dev/null || true
|
||||
echo $$ > "$lock_dir/pid" 2>/dev/null || true
|
||||
return 0
|
||||
fi
|
||||
sleep 1
|
||||
i=$((i + 1))
|
||||
done
|
||||
echo "another rsmon-worker activation is in progress ($lock_dir)" >&2
|
||||
return 1
|
||||
}
|
||||
release_lock() {
|
||||
rm -rf "$lock_dir" 2>/dev/null || true
|
||||
}
|
||||
|
||||
if ! acquire_lock; then
|
||||
exit 1
|
||||
fi
|
||||
# Any early failure (before the rollback trap is armed) still releases the
|
||||
# lock.
|
||||
trap 'release_lock' EXIT
|
||||
|
||||
# --- interrupted-run recovery --------------------------------------------
|
||||
# A previous activation that was killed mid-flight (SSH drop, SIGKILL)
|
||||
# leaves its backup marker behind. Restore that snapshot before the fresh
|
||||
# activation runs, so the host never stays half-activated and the next run
|
||||
# starts from a consistent prior state.
|
||||
unit_enabled=0
|
||||
recover_backup() {
|
||||
if [ ! -f "$backup_dir/.marker" ]; then
|
||||
return 0
|
||||
fi
|
||||
echo "recovering interrupted activation from $backup_dir" >&2
|
||||
if [ -f "$backup_dir/binary" ]; then
|
||||
mkdir -p "$(dirname "$binary")" || true
|
||||
cp -p "$backup_dir/binary" "$binary" || true
|
||||
fi
|
||||
if [ -f "$backup_dir/worker.env" ]; then
|
||||
mkdir -p "$config_dir" || true
|
||||
cp -p "$backup_dir/worker.env" "$env_file" || true
|
||||
fi
|
||||
if [ -n "$unit_file" ] && [ -f "$backup_dir/unit" ]; then
|
||||
mkdir -p "$(dirname "$unit_file")" || true
|
||||
cp -p "$backup_dir/unit" "$unit_file" || true
|
||||
chmod "$unit_mode" "$unit_file" || true
|
||||
fi
|
||||
if [ "$(cat "$backup_dir/.marker" 2>/dev/null || true)" = "1" ]; then
|
||||
unit_enabled=1
|
||||
fi
|
||||
set_unit_enabled "$unit_enabled" || true
|
||||
if [ -f "$backup_dir/binary" ] && [ -f "$backup_dir/worker.env" ] && [ "$no_start" -ne 1 ]; then
|
||||
restart_svc || true
|
||||
fi
|
||||
rm -rf "$backup_dir" || true
|
||||
}
|
||||
recover_backup
|
||||
|
||||
# --- staged binary validation (no state touched) -------------------------
|
||||
# A corrupt or missing staging build aborts here and leaves the prior
|
||||
# install completely untouched (the rollback trap is not armed yet).
|
||||
"$stage" --version >/dev/null
|
||||
|
||||
# --- snapshot the prior install ------------------------------------------
|
||||
rm -rf "$backup_dir"
|
||||
mkdir -p "$backup_dir"
|
||||
had_binary=0
|
||||
had_env=0
|
||||
had_unit=0
|
||||
unit_enabled=0
|
||||
if [ -e "$binary" ]; then
|
||||
cp -p "$binary" "$backup_dir/binary"
|
||||
had_binary=1
|
||||
fi
|
||||
if [ -e "$env_file" ]; then
|
||||
cp -p "$env_file" "$backup_dir/worker.env"
|
||||
had_env=1
|
||||
fi
|
||||
if [ -n "$unit_file" ] && [ -e "$unit_file" ]; then
|
||||
had_unit=1
|
||||
cp -p "$unit_file" "$backup_dir/unit"
|
||||
unit_was_enabled
|
||||
fi
|
||||
# The marker is written only after the snapshot copies so recovery never
|
||||
# sees a partial snapshot; it records whether the prior unit was enabled.
|
||||
printf '%s\n' "$unit_enabled" > "$backup_dir/.marker"
|
||||
|
||||
# --- rollback trap --------------------------------------------------------
|
||||
# Any failure from here on restores the snapshot (preserving binary/env/unit
|
||||
# metadata), restores the prior enable state (or disables a freshly-installed
|
||||
# unit on a fresh install), and brings the prior service back up. The trap
|
||||
# also fires on HUP/INT/TERM so an interrupted run rolls back instead of
|
||||
# leaving a half-activated state.
|
||||
rolled_back=0
|
||||
rollback() {
|
||||
[ "$rolled_back" -eq 1 ] && return 0
|
||||
rolled_back=1
|
||||
echo "rsmon-worker activation failed; restoring the prior install" >&2
|
||||
stop_svc || true
|
||||
rm -f "$binary.new" "$env_file.new" "$unit_file.new" "$pid_file" || true
|
||||
if [ "$had_binary" -eq 1 ]; then
|
||||
cp -p "$backup_dir/binary" "$binary" || true
|
||||
elif [ -e "$binary" ]; then
|
||||
rm -f "$binary" || true
|
||||
fi
|
||||
if [ "$had_env" -eq 1 ]; then
|
||||
cp -p "$backup_dir/worker.env" "$env_file" || true
|
||||
chmod 0600 "$env_file" || true
|
||||
elif [ -e "$env_file" ]; then
|
||||
rm -f "$env_file" || true
|
||||
fi
|
||||
if [ -n "$unit_file" ]; then
|
||||
if [ "$had_unit" -eq 1 ]; then
|
||||
cp -p "$backup_dir/unit" "$unit_file" || true
|
||||
chmod "$unit_mode" "$unit_file" || true
|
||||
set_unit_enabled "$unit_enabled" || true
|
||||
else
|
||||
set_unit_enabled 0 || true
|
||||
rm -f "$unit_file" || true
|
||||
fi
|
||||
fi
|
||||
rm -f "$env_tmp" "$unit_tmp" || true
|
||||
rm -rf "$backup_dir" || true
|
||||
if [ "$had_binary" -eq 1 ] && [ "$had_env" -eq 1 ] && [ "$no_start" -ne 1 ]; then
|
||||
restart_svc || true
|
||||
fi
|
||||
release_lock
|
||||
exit 1
|
||||
}
|
||||
trap rollback EXIT HUP INT TERM
|
||||
|
||||
# --- atomic binary install ------------------------------------------------
|
||||
mkdir -p "$(dirname "$binary")"
|
||||
install -m 0755 "$stage" "$binary.new"
|
||||
mv -f "$binary.new" "$binary"
|
||||
|
||||
# --- atomic env install (secrets, mode 0600) ------------------------------
|
||||
mkdir -p "$config_dir"
|
||||
chmod 0750 "$config_dir"
|
||||
install -m 0600 "$env_tmp" "$env_file.new"
|
||||
mv -f "$env_file.new" "$env_file"
|
||||
rm -f "$env_tmp"
|
||||
|
||||
# --- data directory -------------------------------------------------------
|
||||
mkdir -p "$data_dir" "$data_dir/webapp"
|
||||
chmod 0755 "$data_dir" "$data_dir/webapp"
|
||||
|
||||
# --- service definition ---------------------------------------------------
|
||||
if [ -n "$unit_file" ] && [ -f "$unit_tmp" ]; then
|
||||
mkdir -p "$(dirname "$unit_file")"
|
||||
install -m "$unit_mode" "$unit_tmp" "$unit_file.new"
|
||||
mv -f "$unit_file.new" "$unit_file"
|
||||
rm -f "$unit_tmp"
|
||||
set_unit_enabled 1
|
||||
fi
|
||||
|
||||
# --- start and verify -----------------------------------------------------
|
||||
# A failed start or a worker that does not answer /healthz triggers the
|
||||
# rollback trap.
|
||||
if [ "$no_start" -ne 1 ]; then
|
||||
if ! restart_svc; then
|
||||
echo "rsmon-worker activation failed: process or /healthz verification failed" >&2
|
||||
exit 1
|
||||
fi
|
||||
else
|
||||
stop_svc || true
|
||||
fi
|
||||
|
||||
# --- success --------------------------------------------------------------
|
||||
rm -rf "$backup_dir"
|
||||
rm -f "$env_tmp" "$unit_tmp"
|
||||
release_lock
|
||||
trap - EXIT HUP INT TERM
|
||||
if [ "$no_start" -eq 1 ]; then
|
||||
echo "rsmon-worker activated: binary=$binary supervisor=$supervisor unit=${unit_file:-none} started=no"
|
||||
else
|
||||
echo "rsmon-worker activated: binary=$binary supervisor=$supervisor unit=${unit_file:-none} started=yes"
|
||||
fi
|
||||
exit 0
|
||||
`
|
||||
|
||||
// activateParams are the resolved, controller-validated values fed into
|
||||
// the remote activation script.
|
||||
type activateParams struct {
|
||||
Stage string
|
||||
Binary string
|
||||
ConfigDir string
|
||||
EnvFile string
|
||||
EnvTmp string
|
||||
DataDir string
|
||||
UnitTmp string
|
||||
UnitFile string
|
||||
UnitMode string
|
||||
UnitName string // systemd unit name (rsmon-worker.service)
|
||||
RCName string // rc-service name (rsmon-worker)
|
||||
Supervisor string // systemd | openrc | none
|
||||
NoStart string // 0 or 1
|
||||
}
|
||||
|
||||
// activateWorker performs the work-package-4 activation for an already
|
||||
// staged source build over the live SSH executor. It uses the env bytes
|
||||
// rendered once during option normalization, uploads the env and service
|
||||
// definition into a server-created 0700 temp dir (no /tmp symlink/TOCTOU
|
||||
// attack surface), runs the atomic activation script, and records where
|
||||
// and how the worker was activated. Every remote temp file it creates is
|
||||
// removed on success and failure.
|
||||
func (e *sourceExecutor) activateWorker(opts SourceInstallOptions, res *SourceInstallResult) error {
|
||||
a := opts.Activation
|
||||
name := strings.TrimSpace(a.Name)
|
||||
p := resolvePaths(name)
|
||||
|
||||
envData := opts.activationEnv
|
||||
if len(envData) == 0 {
|
||||
return errors.New("activation environment was not rendered (internal error)")
|
||||
}
|
||||
|
||||
unitContent, unitFile, unitMode := unitContent(res.Detection.InitSystem, p)
|
||||
svcName := strings.TrimSuffix(p.unitName, ".service")
|
||||
|
||||
supOut, err := e.runPlain("sh -c " + shellQuote(runningInitScript))
|
||||
if err != nil {
|
||||
return fmt.Errorf("detect running init system: %w", err)
|
||||
}
|
||||
supervisor := normalizeSupervisor(string(supOut))
|
||||
|
||||
// Create a server-side 0700 temp dir owned by the SSH user, then
|
||||
// upload the env/unit into it. mktemp -d produces an unpredictable,
|
||||
// private path, so a hostile local user cannot pre-create a symlink at
|
||||
// a predictable /tmp name (the classic upload TOCTOU). The token
|
||||
// never appears in argv or logs: it travels only as base64 over the
|
||||
// session stdin and later only inside the mode-0600 env file.
|
||||
out, err := e.runPlain("d=$(mktemp -d /tmp/rsmon-worker-act.XXXXXX) && chmod 0700 \"$d\" && echo \"$d\"")
|
||||
if err != nil {
|
||||
return fmt.Errorf("create secure upload directory: %w", err)
|
||||
}
|
||||
secureDir := strings.TrimSpace(string(out))
|
||||
if secureDir == "" || !strings.HasPrefix(secureDir, "/tmp/") {
|
||||
return errors.New("remote returned an invalid secure upload directory")
|
||||
}
|
||||
envTmp := secureDir + "/worker.env"
|
||||
unitTmp := secureDir + "/unit"
|
||||
if err := uploadBytes(e.client, envData, envTmp, 0o600); err != nil {
|
||||
return fmt.Errorf("upload worker environment: %w", err)
|
||||
}
|
||||
if unitContent != "" {
|
||||
if err := uploadBytes(e.client, []byte(unitContent), unitTmp, 0o644); err != nil {
|
||||
return fmt.Errorf("upload service definition: %w", err)
|
||||
}
|
||||
}
|
||||
defer func() {
|
||||
_ = runRemote(e.client, "rm -rf -- "+shellQuote(secureDir), nil)
|
||||
}()
|
||||
|
||||
noStart := "0"
|
||||
if a.NoStart {
|
||||
noStart = "1"
|
||||
}
|
||||
script := activateScript(activateParams{
|
||||
Stage: res.StageBinary,
|
||||
Binary: p.binary,
|
||||
ConfigDir: p.configDir,
|
||||
EnvFile: p.envFile,
|
||||
EnvTmp: envTmp,
|
||||
DataDir: p.dataDir,
|
||||
UnitTmp: unitTmp,
|
||||
UnitFile: unitFile,
|
||||
UnitMode: unitMode,
|
||||
UnitName: p.unitName,
|
||||
RCName: svcName,
|
||||
Supervisor: supervisor,
|
||||
NoStart: noStart,
|
||||
})
|
||||
if _, err := e.runPrivileged("sh -c " + shellQuote(script)); err != nil {
|
||||
return fmt.Errorf("activate worker service: %w", err)
|
||||
}
|
||||
|
||||
res.Activation = &ActivationResult{
|
||||
Binary: p.binary,
|
||||
ConfigDir: p.configDir,
|
||||
EnvFile: p.envFile,
|
||||
DataDir: p.dataDir,
|
||||
UnitFile: unitFile,
|
||||
UnitName: p.unitName,
|
||||
Supervisor: supervisor,
|
||||
Started: !a.NoStart,
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// validateRenderedEnv runs the strict environment-file parser over the
|
||||
// rendered env bytes in memory so the exact bytes written remotely pass
|
||||
// the same gate as a user-supplied file, without re-reading a file.
|
||||
func validateRenderedEnv(data []byte) error {
|
||||
_, err := parseEnvironmentContent(data)
|
||||
return err
|
||||
}
|
||||
|
||||
// openrcInitFor renders the native OpenRC init script for an instance. It
|
||||
// mirrors the hardening of the systemd unit (data dir, env file, webapp
|
||||
// data dir) using OpenRC conventions; it is written for real hosts where
|
||||
// openrc is the running init, while containers that cannot run openrc
|
||||
// fall back to the embedded supervisor.
|
||||
func openrcInitFor(p paths) string {
|
||||
description := "RSMon distributed monitoring worker"
|
||||
svcName := strings.TrimSuffix(p.unitName, ".service")
|
||||
if p.name != "" {
|
||||
description += " (" + p.name + ")"
|
||||
}
|
||||
return fmt.Sprintf(`#!/sbin/openrc-run
|
||||
# Managed by the rsmon-worker source installer; do not edit by hand.
|
||||
|
||||
name=%s
|
||||
description=%s
|
||||
command=%s
|
||||
command_background=true
|
||||
pidfile=%s/worker.pid
|
||||
output_log=%s/worker.log
|
||||
error_log=%s/worker.log
|
||||
|
||||
depend() {
|
||||
need net
|
||||
}
|
||||
|
||||
start_pre() {
|
||||
checkpath --directory --mode 0755 --owner root:root %s %s/webapp
|
||||
if [ -f %s ]; then
|
||||
# OpenRC runs the command with the init script's environment, so
|
||||
# the env-file variables must be exported (set -a) or the worker
|
||||
# would start without them.
|
||||
set -a
|
||||
. %s
|
||||
set +a
|
||||
fi
|
||||
export HOME=%s
|
||||
export RSMON_WEBAPP_DATA_DIR=%s/webapp
|
||||
}
|
||||
`, svcName, description, p.binary, p.dataDir, p.dataDir, p.dataDir,
|
||||
p.dataDir, p.dataDir, p.envFile, p.envFile, p.dataDir, p.dataDir)
|
||||
}
|
||||
438
internal/installer/sourceactivate_shell_test.go
Обычный файл
438
internal/installer/sourceactivate_shell_test.go
Обычный файл
@@ -0,0 +1,438 @@
|
||||
package installer
|
||||
|
||||
import (
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// This file exercises the rendered activation script against stub
|
||||
// systemd/OpenRC implementations ("targeted shell fixture tests"). Real
|
||||
// systemd and OpenRC cannot run inside the Docker/OpenSSH harness (the
|
||||
// container PID 1 is sshd), so these tests execute the actual remote
|
||||
// script on the host with fake `systemctl`, `rc-service`, and `rc-update`
|
||||
// binaries that simulate an init-managed host. This covers the unit
|
||||
// install/enable, the supervisor-driven restart, the `is-active`-based
|
||||
// health loop, the enable-state rollback, the activation lock, and the
|
||||
// interrupted-run recovery paths that the E2E fixtures cannot reach.
|
||||
//
|
||||
// Residual limitation: the stub init tools verify command flow and state
|
||||
// transitions, not the real systemd/OpenRC unit semantics; a real
|
||||
// init-system smoke test remains out of scope for the container harness.
|
||||
|
||||
// systemctlStub records every invocation and simulates unit state in a
|
||||
// fake state directory: `is-active`/`is-enabled` consult marker files,
|
||||
// `restart` marks the unit active (or fails when the fail-restart marker
|
||||
// is present), and `enable`/`disable` toggle the enabled marker.
|
||||
const systemctlStub = `#!/bin/sh
|
||||
echo "$*" >> "$SYSCTL_LOG"
|
||||
action="$1"
|
||||
shift
|
||||
unit=""
|
||||
for a in "$@"; do
|
||||
case "$a" in --*) continue ;; esac
|
||||
unit="$a"
|
||||
break
|
||||
done
|
||||
case "$action" in
|
||||
is-active) [ -f "$FAKE_SYSTEMD_DIR/$unit.active" ] || exit 3 ;;
|
||||
is-enabled) [ -f "$FAKE_SYSTEMD_DIR/$unit.enabled" ] || exit 1 ;;
|
||||
enable) touch "$FAKE_SYSTEMD_DIR/$unit.enabled"; exit 0 ;;
|
||||
disable) rm -f "$FAKE_SYSTEMD_DIR/$unit.enabled"; exit 0 ;;
|
||||
daemon-reload) exit 0 ;;
|
||||
restart) [ -f "$FAKE_SYSTEMD_DIR/fail-restart" ] && exit 1
|
||||
touch "$FAKE_SYSTEMD_DIR/$unit.active"; exit 0 ;;
|
||||
stop) rm -f "$FAKE_SYSTEMD_DIR/$unit.active"; exit 0 ;;
|
||||
*) exit 0 ;;
|
||||
esac
|
||||
`
|
||||
|
||||
// rcServiceStub simulates OpenRC service state; rcUpdateStub simulates
|
||||
// runlevel enablement.
|
||||
const rcServiceStub = `#!/bin/sh
|
||||
echo "$*" >> "$RC_LOG"
|
||||
svc="$1"
|
||||
action="$2"
|
||||
case "$action" in
|
||||
restart) [ -f "$FAKE_RC_DIR/fail-restart" ] && exit 1
|
||||
touch "$FAKE_RC_DIR/$svc.active"; exit 0 ;;
|
||||
status) [ -f "$FAKE_RC_DIR/$svc.active" ] || exit 1 ;;
|
||||
stop) rm -f "$FAKE_RC_DIR/$svc.active"; exit 0 ;;
|
||||
*) exit 0 ;;
|
||||
esac
|
||||
`
|
||||
|
||||
const rcUpdateStub = `#!/bin/sh
|
||||
echo "$*" >> "$RCU_LOG"
|
||||
case "$1" in
|
||||
add) touch "$FAKE_RC_DIR/$2.enabled"; exit 0 ;;
|
||||
del) rm -f "$FAKE_RC_DIR/$2.enabled"; exit 0 ;;
|
||||
*) exit 0 ;;
|
||||
esac
|
||||
`
|
||||
|
||||
// workerStub stands in for the built worker binary: it satisfies the
|
||||
// `--version` validation and the `/healthz` liveness probe.
|
||||
const workerStub = `#!/bin/sh
|
||||
if [ "$1" = "--version" ]; then
|
||||
echo "rsmon-worker version=dev commit=deadbeefdead buildDate=2026-01-01T00:00:00Z"
|
||||
exit 0
|
||||
fi
|
||||
if [ "$1" = "liveness" ]; then
|
||||
exit 0
|
||||
fi
|
||||
exit 1
|
||||
`
|
||||
|
||||
// initShellFixture builds a temp "host" with stub init tools and a stub
|
||||
// worker, plus the uploaded env/unit the activation script consumes.
|
||||
type initShellFixture struct {
|
||||
root string
|
||||
binDir string
|
||||
sysdDir string
|
||||
rcDir string
|
||||
|
||||
sysctlLog string
|
||||
rcLog string
|
||||
rcuLog string
|
||||
|
||||
params activateParams
|
||||
}
|
||||
|
||||
func writeExec(t *testing.T, path, content string) {
|
||||
t.Helper()
|
||||
if err := os.WriteFile(path, []byte(content), 0o700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func newInitShellFixture(t *testing.T, supervisor string) *initShellFixture {
|
||||
t.Helper()
|
||||
root := t.TempDir()
|
||||
fx := &initShellFixture{
|
||||
root: root,
|
||||
binDir: filepath.Join(root, "bin"),
|
||||
sysdDir: filepath.Join(root, "sysd"),
|
||||
rcDir: filepath.Join(root, "rc"),
|
||||
sysctlLog: filepath.Join(root, "sysctl.log"),
|
||||
rcLog: filepath.Join(root, "rc.log"),
|
||||
rcuLog: filepath.Join(root, "rcu.log"),
|
||||
}
|
||||
for _, d := range []string{
|
||||
fx.binDir, fx.sysdDir, fx.rcDir, filepath.Join(root, "upload"),
|
||||
filepath.Join(root, "usr/local/bin"), filepath.Join(root, "etc/rsmon-worker"),
|
||||
} {
|
||||
if err := os.MkdirAll(d, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
writeExec(t, filepath.Join(fx.binDir, "systemctl"), systemctlStub)
|
||||
writeExec(t, filepath.Join(fx.binDir, "rc-service"), rcServiceStub)
|
||||
writeExec(t, filepath.Join(fx.binDir, "rc-update"), rcUpdateStub)
|
||||
|
||||
stage := filepath.Join(root, "usr/local/bin/rsmon-worker.stage")
|
||||
writeExec(t, stage, workerStub)
|
||||
|
||||
envTmp := filepath.Join(root, "upload", "worker.env")
|
||||
unitTmp := filepath.Join(root, "upload", "unit")
|
||||
if err := os.WriteFile(envTmp, []byte(
|
||||
"RSMON_URL=https://rsmon.ru\nRSMON_TOKEN=shell-test-token\nWORKER_HOST=127.0.0.1\nWORKER_PORT=27401\n",
|
||||
), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
unitFile := filepath.Join(root, "etc/systemd/system/rsmon-worker.service")
|
||||
unitMode := "0644"
|
||||
if supervisor == "openrc" {
|
||||
unitFile = filepath.Join(root, "etc/init.d/rsmon-worker")
|
||||
unitMode = "0755"
|
||||
if err := os.MkdirAll(filepath.Dir(unitFile), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(unitTmp, []byte("# fake openrc unit\n"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
} else {
|
||||
if err := os.MkdirAll(filepath.Dir(unitFile), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(unitTmp, []byte("[Unit]\n# fake systemd unit\n"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
dataDir := filepath.Join(root, "var/lib/rsmon-worker")
|
||||
fx.params = activateParams{
|
||||
Stage: stage,
|
||||
Binary: filepath.Join(root, "usr/local/bin/rsmon-worker"),
|
||||
ConfigDir: filepath.Join(root, "etc/rsmon-worker"),
|
||||
EnvFile: filepath.Join(root, "etc/rsmon-worker/worker.env"),
|
||||
EnvTmp: envTmp,
|
||||
DataDir: dataDir,
|
||||
UnitTmp: unitTmp,
|
||||
UnitFile: unitFile,
|
||||
UnitMode: unitMode,
|
||||
UnitName: "rsmon-worker.service",
|
||||
RCName: "rsmon-worker",
|
||||
Supervisor: supervisor,
|
||||
NoStart: "0",
|
||||
}
|
||||
return fx
|
||||
}
|
||||
|
||||
// run executes the rendered activation script with the stub init tools
|
||||
// first in PATH.
|
||||
func (fx *initShellFixture) run(t *testing.T) (string, error) {
|
||||
t.Helper()
|
||||
script := activateScript(fx.params)
|
||||
cmd := exec.Command("sh", "-c", script)
|
||||
cmd.Env = append(
|
||||
os.Environ(),
|
||||
"PATH="+fx.binDir+":"+os.Getenv("PATH"),
|
||||
"FAKE_SYSTEMD_DIR="+fx.sysdDir,
|
||||
"FAKE_RC_DIR="+fx.rcDir,
|
||||
"SYSCTL_LOG="+fx.sysctlLog,
|
||||
"RC_LOG="+fx.rcLog,
|
||||
"RCU_LOG="+fx.rcuLog,
|
||||
)
|
||||
out, err := cmd.CombinedOutput()
|
||||
return string(out), err
|
||||
}
|
||||
|
||||
func (fx *initShellFixture) log(t *testing.T, path string) string {
|
||||
t.Helper()
|
||||
b, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
return string(b)
|
||||
}
|
||||
|
||||
func (fx *initShellFixture) assertNoLeftovers(t *testing.T) {
|
||||
t.Helper()
|
||||
for _, name := range []string{".rsmon-backup", ".rsmon-activate.lock"} {
|
||||
if _, err := os.Stat(filepath.Join(fx.params.DataDir, name)); !os.IsNotExist(err) {
|
||||
t.Fatalf("%s left behind after activation", name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestActivateShellSystemd runs the real activation script against a stub
|
||||
// systemd host and verifies the init-managed flow: the unit is installed
|
||||
// and enabled, restart goes through systemctl, and the health loop uses
|
||||
// `systemctl is-active` (no pid file is ever created).
|
||||
func TestActivateShellSystemd(t *testing.T) {
|
||||
fx := newInitShellFixture(t, "systemd")
|
||||
out, err := fx.run(t)
|
||||
if err != nil {
|
||||
t.Fatalf("systemd activation failed: %v\n%s", err, out)
|
||||
}
|
||||
if !strings.Contains(out, "rsmon-worker activated") {
|
||||
t.Fatalf("no activation success line:\n%s", out)
|
||||
}
|
||||
unit, err := os.ReadFile(fx.params.UnitFile)
|
||||
if err != nil {
|
||||
t.Fatalf("unit not installed: %v", err)
|
||||
}
|
||||
if !strings.Contains(string(unit), "[Unit]") {
|
||||
t.Fatalf("installed unit has wrong content: %q", unit)
|
||||
}
|
||||
st, err := os.Stat(fx.params.EnvFile)
|
||||
if err != nil {
|
||||
t.Fatalf("env not installed: %v", err)
|
||||
}
|
||||
if st.Mode().Perm() != 0o600 {
|
||||
t.Fatalf("env mode = %v, want 0600", st.Mode().Perm())
|
||||
}
|
||||
log := fx.log(t, fx.sysctlLog)
|
||||
for _, want := range []string{
|
||||
"enable rsmon-worker.service",
|
||||
"restart rsmon-worker.service",
|
||||
"is-active --quiet rsmon-worker.service",
|
||||
} {
|
||||
if !strings.Contains(log, want) {
|
||||
t.Fatalf("systemctl log missing %q:\n%s", want, log)
|
||||
}
|
||||
}
|
||||
// The health loop used systemctl is-active, never the embedded
|
||||
// supervisor's pid file.
|
||||
if _, err := os.Stat(filepath.Join(fx.params.DataDir, "worker.pid")); !os.IsNotExist(err) {
|
||||
t.Fatalf("systemd activation created a pid file; the health loop must use systemctl is-active")
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(fx.sysdDir, "rsmon-worker.service.enabled")); err != nil {
|
||||
t.Fatalf("unit not enabled: %v", err)
|
||||
}
|
||||
fx.assertNoLeftovers(t)
|
||||
}
|
||||
|
||||
// TestActivateShellOpenRC runs the real activation script against a stub
|
||||
// OpenRC host: the init script is installed executable, the runlevel
|
||||
// enablement and rc-service restart/status are used, and the health loop
|
||||
// goes through rc-service (no pid file).
|
||||
func TestActivateShellOpenRC(t *testing.T) {
|
||||
fx := newInitShellFixture(t, "openrc")
|
||||
out, err := fx.run(t)
|
||||
if err != nil {
|
||||
t.Fatalf("openrc activation failed: %v\n%s", err, out)
|
||||
}
|
||||
if !strings.Contains(out, "rsmon-worker activated") {
|
||||
t.Fatalf("no activation success line:\n%s", out)
|
||||
}
|
||||
st, err := os.Stat(fx.params.UnitFile)
|
||||
if err != nil {
|
||||
t.Fatalf("openrc init not installed: %v", err)
|
||||
}
|
||||
if st.Mode().Perm() != 0o755 {
|
||||
t.Fatalf("openrc init mode = %v, want 0755", st.Mode().Perm())
|
||||
}
|
||||
rcLog := fx.log(t, fx.rcLog)
|
||||
for _, want := range []string{"rsmon-worker restart", "rsmon-worker status"} {
|
||||
if !strings.Contains(rcLog, want) {
|
||||
t.Fatalf("rc-service log missing %q:\n%s", want, rcLog)
|
||||
}
|
||||
}
|
||||
rcuLog := fx.log(t, fx.rcuLog)
|
||||
if !strings.Contains(rcuLog, "add rsmon-worker default") {
|
||||
t.Fatalf("rc-update add not issued:\n%s", rcuLog)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(fx.rcDir, "rsmon-worker.enabled")); err != nil {
|
||||
t.Fatalf("service not enabled in the runlevel: %v", err)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(fx.params.DataDir, "worker.pid")); !os.IsNotExist(err) {
|
||||
t.Fatalf("openrc activation created a pid file; the health loop must use rc-service status")
|
||||
}
|
||||
fx.assertNoLeftovers(t)
|
||||
}
|
||||
|
||||
// TestActivateShellSystemdRollbackFreshDisablesUnit proves a failed
|
||||
// activation on a fresh host disables the newly-enabled unit and removes
|
||||
// the unit file (no dangling enablement).
|
||||
func TestActivateShellSystemdRollbackFreshDisablesUnit(t *testing.T) {
|
||||
fx := newInitShellFixture(t, "systemd")
|
||||
if err := os.WriteFile(filepath.Join(fx.sysdDir, "fail-restart"), []byte(""), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
out, err := fx.run(t)
|
||||
if err == nil {
|
||||
t.Fatal("activation succeeded despite restart failure")
|
||||
}
|
||||
if !strings.Contains(out, "start failure") {
|
||||
t.Fatalf("no start-failure classification:\n%s", out)
|
||||
}
|
||||
log := fx.log(t, fx.sysctlLog)
|
||||
if !strings.Contains(log, "disable rsmon-worker.service") {
|
||||
t.Fatalf("fresh rollback did not disable the unit:\n%s", log)
|
||||
}
|
||||
if _, err := os.Stat(fx.params.UnitFile); !os.IsNotExist(err) {
|
||||
t.Fatal("unit file left behind after fresh rollback")
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(fx.sysdDir, "rsmon-worker.service.enabled")); !os.IsNotExist(err) {
|
||||
t.Fatal("unit still enabled after fresh rollback")
|
||||
}
|
||||
fx.assertNoLeftovers(t)
|
||||
}
|
||||
|
||||
// TestActivateShellSystemdRollbackRestoresEnableState proves a failed
|
||||
// activation on a host with a prior enabled unit restores the unit
|
||||
// byte-for-byte and re-applies the prior enabled state.
|
||||
func TestActivateShellSystemdRollbackRestoresEnableState(t *testing.T) {
|
||||
fx := newInitShellFixture(t, "systemd")
|
||||
if err := os.WriteFile(fx.params.UnitFile, []byte("PRIOR-UNIT-CONTENT"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(fx.sysdDir, "rsmon-worker.service.enabled"), []byte(""), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(fx.sysdDir, "fail-restart"), []byte(""), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
out, err := fx.run(t)
|
||||
if err == nil {
|
||||
t.Fatal("activation succeeded despite restart failure")
|
||||
}
|
||||
if !strings.Contains(out, "start failure") {
|
||||
t.Fatalf("no start-failure classification:\n%s", out)
|
||||
}
|
||||
unit, err := os.ReadFile(fx.params.UnitFile)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if string(unit) != "PRIOR-UNIT-CONTENT" {
|
||||
t.Fatalf("prior unit not restored: %q", unit)
|
||||
}
|
||||
log := fx.log(t, fx.sysctlLog)
|
||||
if !strings.Contains(log, "enable rsmon-worker.service") {
|
||||
t.Fatalf("prior enabled state not re-applied:\n%s", log)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(fx.sysdDir, "rsmon-worker.service.enabled")); err != nil {
|
||||
t.Fatalf("unit not re-enabled after rollback: %v", err)
|
||||
}
|
||||
fx.assertNoLeftovers(t)
|
||||
}
|
||||
|
||||
// TestActivateShellStaleLockBroken proves a leftover lock from a killed
|
||||
// run (dead pid) is broken automatically and the activation proceeds.
|
||||
func TestActivateShellStaleLockBroken(t *testing.T) {
|
||||
fx := newInitShellFixture(t, "systemd")
|
||||
lockDir := filepath.Join(fx.params.DataDir, ".rsmon-activate.lock")
|
||||
if err := os.MkdirAll(lockDir, 0o700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(lockDir, "pid"), []byte("999999\n"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
out, err := fx.run(t)
|
||||
if err != nil {
|
||||
t.Fatalf("activation failed with a stale lock: %v\n%s", err, out)
|
||||
}
|
||||
if !strings.Contains(out, "rsmon-worker activated") {
|
||||
t.Fatalf("no activation success line:\n%s", out)
|
||||
}
|
||||
if _, err := os.Stat(lockDir); !os.IsNotExist(err) {
|
||||
t.Fatal("lock not released after activation")
|
||||
}
|
||||
}
|
||||
|
||||
// TestActivateShellRecoveryFromInterruptedBackup proves the interrupted-run
|
||||
// recovery: a leftover backup marker from a killed activation is restored
|
||||
// before the fresh run, so a later validation failure still leaves the
|
||||
// recovered prior install in place.
|
||||
func TestActivateShellRecoveryFromInterruptedBackup(t *testing.T) {
|
||||
fx := newInitShellFixture(t, "none")
|
||||
backupDir := filepath.Join(fx.params.DataDir, ".rsmon-backup")
|
||||
if err := os.MkdirAll(backupDir, 0o700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(backupDir, "binary"), []byte("RECOVERED-BINARY"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(backupDir, "worker.env"), []byte("RSMON_URL=https://rsmon.ru\nRSMON_TOKEN=test\n"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(backupDir, ".marker"), []byte("0"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Corrupt the stage so the fresh activation fails at validation (before
|
||||
// the fresh snapshot), proving the recovered files were already in place.
|
||||
if err := os.WriteFile(fx.params.Stage, []byte("not a script"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
out, err := fx.run(t)
|
||||
if err == nil {
|
||||
t.Fatal("activation succeeded with a corrupt stage")
|
||||
}
|
||||
if !strings.Contains(out, "recovering interrupted activation") {
|
||||
t.Fatalf("recovery did not run:\n%s", out)
|
||||
}
|
||||
b, err := os.ReadFile(fx.params.Binary)
|
||||
if err != nil {
|
||||
t.Fatalf("recovered binary not restored: %v", err)
|
||||
}
|
||||
if string(b) != "RECOVERED-BINARY" {
|
||||
t.Fatalf("recovered binary content = %q", b)
|
||||
}
|
||||
if _, err := os.Stat(backupDir); !os.IsNotExist(err) {
|
||||
t.Fatal("backup not consumed by recovery")
|
||||
}
|
||||
}
|
||||
409
internal/installer/sourceactivate_test.go
Обычный файл
409
internal/installer/sourceactivate_test.go
Обычный файл
@@ -0,0 +1,409 @@
|
||||
package installer
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"rocketgit.ru/rsmon/worker/internal/sshinstall"
|
||||
)
|
||||
|
||||
func TestActivateScriptMarkers(t *testing.T) {
|
||||
script := activateScript(activateParams{
|
||||
Stage: "/opt/rsmon-worker-src/rsmon-worker",
|
||||
Binary: "/usr/local/bin/rsmon-worker",
|
||||
ConfigDir: "/etc/rsmon-worker",
|
||||
EnvFile: "/etc/rsmon-worker/worker.env",
|
||||
EnvTmp: "/tmp/rsmon-worker-abc.env",
|
||||
DataDir: "/var/lib/rsmon-worker",
|
||||
UnitTmp: "/tmp/rsmon-worker-abc.service",
|
||||
UnitFile: "/etc/systemd/system/rsmon-worker.service",
|
||||
UnitMode: "0644",
|
||||
UnitName: "rsmon-worker.service",
|
||||
RCName: "rsmon-worker",
|
||||
Supervisor: "none",
|
||||
NoStart: "0",
|
||||
})
|
||||
for _, want := range []string{
|
||||
"set -eu",
|
||||
// staged binary validated before any state is touched
|
||||
"\"$stage\" --version >/dev/null",
|
||||
// snapshot + rollback
|
||||
"backup_dir=\"$data_dir/.rsmon-backup\"",
|
||||
"rm -rf \"$backup_dir\"",
|
||||
"cp -p \"$binary\" \"$backup_dir/binary\"",
|
||||
"cp -p \"$env_file\" \"$backup_dir/worker.env\"",
|
||||
"printf '%s\\n' \"$unit_enabled\" > \"$backup_dir/.marker\"",
|
||||
"trap rollback EXIT HUP INT TERM",
|
||||
"restoring the prior install",
|
||||
// atomic installs with the right perms
|
||||
"install -m 0755 \"$stage\" \"$binary.new\"",
|
||||
"mv -f \"$binary.new\" \"$binary\"",
|
||||
"install -m 0600 \"$env_tmp\" \"$env_file.new\"",
|
||||
"mv -f \"$env_file.new\" \"$env_file\"",
|
||||
"chmod 0750 \"$config_dir\"",
|
||||
"mkdir -p \"$data_dir\" \"$data_dir/webapp\"",
|
||||
"install -m \"$unit_mode\" \"$unit_tmp\" \"$unit_file.new\"",
|
||||
"mv -f \"$unit_file.new\" \"$unit_file\"",
|
||||
// supervisor + pid + health verification
|
||||
"pid_file=\"$data_dir/worker.pid\"",
|
||||
"env -i PATH=\"/usr/bin:/bin:/sbin:/usr/sbin\"",
|
||||
"RSMON_WORKER_ENV_FILE=\"$env_file\"",
|
||||
". \"$RSMON_WORKER_ENV_FILE\"",
|
||||
"exec \"$@\"",
|
||||
"nohup \"$1\" >>\"$2\" 2>&1 & echo $! > \"$3\"",
|
||||
"run_env \"$binary\" liveness >/dev/null 2>&1",
|
||||
"while [ \"$i\" -lt 30 ]; do",
|
||||
"rsmon-worker activated:",
|
||||
// activation lock + interrupted-run recovery
|
||||
"lock_dir=\"$data_dir/.rsmon-activate.lock\"",
|
||||
"acquire_lock",
|
||||
"another rsmon-worker activation is in progress",
|
||||
"recovering interrupted activation from $backup_dir",
|
||||
".marker",
|
||||
// pid belongs to the expected binary before kill
|
||||
"readlink \"/proc/$1/exe\"",
|
||||
"\"$binary (deleted)\"",
|
||||
"process_is_worker \"$pid\"",
|
||||
// rollback preserves metadata and enable state
|
||||
"cp -p \"$backup_dir/binary\" \"$binary\"",
|
||||
"cp -p \"$backup_dir/worker.env\" \"$env_file\"",
|
||||
"set_unit_enabled \"$unit_enabled\"",
|
||||
"systemctl is-enabled",
|
||||
// temp/backup cleanup on success
|
||||
"rm -rf \"$backup_dir\"",
|
||||
"release_lock",
|
||||
"trap - EXIT HUP INT TERM",
|
||||
} {
|
||||
if !strings.Contains(script, want) {
|
||||
t.Fatalf("activate script missing %q:\n%s", want, script)
|
||||
}
|
||||
}
|
||||
// The rollback must be armed only after the snapshot so a corrupt
|
||||
// staging binary (validated first) never triggers a destructive
|
||||
// rollback of the prior install.
|
||||
if !strings.Contains(script, "\"$stage\" --version") || !strings.Contains(script, "trap rollback EXIT HUP INT TERM") {
|
||||
t.Fatalf("activate script must validate the staged binary before arming rollback:\n%s", script)
|
||||
}
|
||||
stageCheck := strings.Index(script, "\"$stage\" --version")
|
||||
trapIdx := strings.Index(script, "trap rollback EXIT HUP INT TERM")
|
||||
if stageCheck < 0 || trapIdx < stageCheck {
|
||||
t.Fatalf("staged-binary validation must precede the rollback trap:\n%s", script)
|
||||
}
|
||||
}
|
||||
|
||||
func TestActivateScriptSystemdSupervisor(t *testing.T) {
|
||||
script := activateScript(activateParams{
|
||||
Stage: "/opt/rsmon-worker-src/rsmon-worker",
|
||||
Binary: "/usr/local/bin/rsmon-worker",
|
||||
ConfigDir: "/etc/rsmon-worker",
|
||||
EnvFile: "/etc/rsmon-worker/worker.env",
|
||||
EnvTmp: "/tmp/e.env",
|
||||
DataDir: "/var/lib/rsmon-worker",
|
||||
UnitTmp: "/tmp/u.service",
|
||||
UnitFile: "/etc/systemd/system/rsmon-worker.service",
|
||||
UnitMode: "0644",
|
||||
UnitName: "rsmon-worker.service",
|
||||
RCName: "rsmon-worker",
|
||||
Supervisor: "systemd",
|
||||
NoStart: "0",
|
||||
})
|
||||
for _, want := range []string{
|
||||
"supervisor=systemd",
|
||||
"systemctl daemon-reload",
|
||||
"systemctl restart \"$unit_name\"",
|
||||
"systemctl is-active --quiet \"$unit_name\"",
|
||||
"systemctl enable \"$unit_name\"",
|
||||
"svc_active()",
|
||||
"start failure: systemctl restart $unit_name failed",
|
||||
"start failure: worker service is not active",
|
||||
} {
|
||||
if !strings.Contains(script, want) {
|
||||
t.Fatalf("systemd supervisor missing %q:\n%s", want, script)
|
||||
}
|
||||
}
|
||||
// The systemd health loop must use `systemctl is-active`, never the
|
||||
// embedded supervisor's pid file.
|
||||
if !strings.Contains(script, "systemd) systemctl is-active --quiet \"$unit_name\"") {
|
||||
t.Fatalf("systemd health loop must use systemctl is-active:\n%s", script)
|
||||
}
|
||||
}
|
||||
|
||||
func TestActivateScriptOpenRCSupervisor(t *testing.T) {
|
||||
script := activateScript(activateParams{
|
||||
Stage: "/opt/rsmon-worker-src/rsmon-worker",
|
||||
Binary: "/usr/local/bin/rsmon-worker",
|
||||
ConfigDir: "/etc/rsmon-worker",
|
||||
EnvFile: "/etc/rsmon-worker/worker.env",
|
||||
EnvTmp: "/tmp/e.env",
|
||||
DataDir: "/var/lib/rsmon-worker",
|
||||
UnitTmp: "/tmp/u",
|
||||
UnitFile: "/etc/init.d/rsmon-worker",
|
||||
UnitMode: "0755",
|
||||
UnitName: "rsmon-worker.service",
|
||||
RCName: "rsmon-worker",
|
||||
Supervisor: "openrc",
|
||||
NoStart: "0",
|
||||
})
|
||||
for _, want := range []string{
|
||||
"rc-service \"$rc_name\" restart",
|
||||
"rc-service \"$rc_name\" status",
|
||||
"rc-update add \"$rc_name\" default",
|
||||
"start failure: rc-service restart $rc_name failed",
|
||||
"rc-update del \"$rc_name\" default",
|
||||
} {
|
||||
if !strings.Contains(script, want) {
|
||||
t.Fatalf("openrc supervisor missing %q:\n%s", want, script)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestActivateScriptNoStartSkipsRestart(t *testing.T) {
|
||||
script := activateScript(activateParams{
|
||||
Stage: "/opt/rsmon-worker-src/rsmon-worker", Binary: "/usr/local/bin/rsmon-worker",
|
||||
ConfigDir: "/etc/rsmon-worker", EnvFile: "/etc/rsmon-worker/worker.env", EnvTmp: "/tmp/e.env",
|
||||
DataDir: "/var/lib/rsmon-worker", UnitTmp: "", UnitFile: "", UnitMode: "",
|
||||
UnitName: "rsmon-worker.service", RCName: "rsmon-worker", Supervisor: "none", NoStart: "1",
|
||||
})
|
||||
if !strings.Contains(script, "[ \"$no_start\" -ne 1 ]") {
|
||||
t.Fatalf("no-start gate missing:\n%s", script)
|
||||
}
|
||||
if !strings.Contains(script, "started=no") {
|
||||
t.Fatalf("no-start summary missing:\n%s", script)
|
||||
}
|
||||
}
|
||||
|
||||
func TestActivateScriptNeverContainsSecrets(t *testing.T) {
|
||||
const secret = "super-secret-token-value"
|
||||
params := activateParams{
|
||||
Stage: "/opt/rsmon-worker-src/rsmon-worker", Binary: "/usr/local/bin/rsmon-worker",
|
||||
ConfigDir: "/etc/rsmon-worker", EnvFile: "/etc/rsmon-worker/worker.env", EnvTmp: "/tmp/e.env",
|
||||
DataDir: "/var/lib/rsmon-worker", UnitTmp: "/tmp/u", UnitFile: "/etc/systemd/system/rsmon-worker.service",
|
||||
UnitMode: "0644", UnitName: "rsmon-worker.service", RCName: "rsmon-worker",
|
||||
Supervisor: "systemd", NoStart: "0",
|
||||
}
|
||||
script := activateScript(params)
|
||||
if strings.Contains(script, secret) {
|
||||
t.Fatalf("activate script contains a secret:\n%s", script)
|
||||
}
|
||||
// The env file is referenced by path; its contents (the secrets) are
|
||||
// only sourced at runtime and never echoed.
|
||||
for _, want := range []string{"RSMON_TOKEN=", secret} {
|
||||
if strings.Contains(script, want) {
|
||||
t.Fatalf("activate script must not embed env contents (%q):\n%s", want, script)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenRCInit(t *testing.T) {
|
||||
unit := openrcInitFor(resolvePaths(""))
|
||||
for _, want := range []string{
|
||||
"#!/sbin/openrc-run",
|
||||
"name=rsmon-worker",
|
||||
"description=RSMon distributed monitoring worker",
|
||||
"command=/usr/local/bin/rsmon-worker",
|
||||
"command_background=true",
|
||||
"pidfile=/var/lib/rsmon-worker/worker.pid",
|
||||
"output_log=/var/lib/rsmon-worker/worker.log",
|
||||
"need net",
|
||||
". /etc/rsmon-worker/worker.env",
|
||||
"set -a",
|
||||
"set +a",
|
||||
"RSMON_WEBAPP_DATA_DIR=/var/lib/rsmon-worker/webapp",
|
||||
} {
|
||||
if !strings.Contains(unit, want) {
|
||||
t.Fatalf("openrc init missing %q:\n%s", want, unit)
|
||||
}
|
||||
}
|
||||
named := openrcInitFor(resolvePaths("edge"))
|
||||
for _, want := range []string{
|
||||
"name=rsmon-worker-edge", "command=/usr/local/bin/rsmon-worker-edge",
|
||||
"pidfile=/var/lib/rsmon-worker-edge/worker.pid", "/etc/rsmon-worker-edge/worker.env", "(edge)",
|
||||
} {
|
||||
if !strings.Contains(named, want) {
|
||||
t.Fatalf("named openrc init missing %q:\n%s", want, named)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunningInitScript(t *testing.T) {
|
||||
for _, want := range []string{"set -eu", "/run/systemd/system", "systemctl", "/run/openrc/softlevel", "rc-service", "echo none"} {
|
||||
if !strings.Contains(runningInitScript, want) {
|
||||
t.Fatalf("running-init script missing %q:\n%s", want, runningInitScript)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeSupervisor(t *testing.T) {
|
||||
for _, in := range []string{"systemd", "openrc", "systemd\n", " openrc "} {
|
||||
want := strings.TrimSpace(in)
|
||||
if got := normalizeSupervisor(in); got != want {
|
||||
t.Fatalf("normalizeSupervisor(%q) = %q, want %q", in, got, want)
|
||||
}
|
||||
}
|
||||
for _, in := range []string{"", "none", "sysvinit", " "} {
|
||||
if got := normalizeSupervisor(in); got != "none" {
|
||||
t.Fatalf("normalizeSupervisor(%q) = %q, want none", in, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestUnitContent(t *testing.T) {
|
||||
p := resolvePaths("")
|
||||
systemd, file, mode := unitContent(sshinstall.InitSystemd, p)
|
||||
if systemd == "" || file != "/etc/systemd/system/rsmon-worker.service" || mode != "0644" {
|
||||
t.Fatalf("systemd unit content = %q, %q, %q", systemd, file, mode)
|
||||
}
|
||||
if !strings.Contains(systemd, "ExecStart=/usr/local/bin/rsmon-worker\n") {
|
||||
t.Fatalf("systemd unit not the classic hardened unit:\n%s", systemd)
|
||||
}
|
||||
openrc, file, mode := unitContent(sshinstall.InitOpenRC, p)
|
||||
if openrc == "" || file != "/etc/init.d/rsmon-worker" || mode != "0755" {
|
||||
t.Fatalf("openrc unit content = %q, %q, %q", openrc, file, mode)
|
||||
}
|
||||
if content, file, mode := unitContent(sshinstall.InitUnknown, p); content != "" || file != "" || mode != "" {
|
||||
t.Fatalf("unknown-init unit content = %q, %q, %q, want empty (no-service gate)", content, file, mode)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRenderEnvActivation(t *testing.T) {
|
||||
t.Setenv("RSMON_URL", "")
|
||||
t.Setenv("RSMON_TOKEN", "")
|
||||
t.Setenv("WORKER_HOST", "")
|
||||
t.Setenv("WORKER_PORT", "")
|
||||
t.Setenv("WORKER_LOGIN", "")
|
||||
t.Setenv("WORKER_PASSWORD", "")
|
||||
|
||||
data, err := (ActivationOptions{Activate: true, URL: "https://rsmon.ru", Token: "secret"}).renderEnv()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, want := range []string{"RSMON_URL=https://rsmon.ru\n", "RSMON_TOKEN=secret\n", "WORKER_HOST=127.0.0.1\n", "WORKER_PORT=27401\n"} {
|
||||
if !strings.Contains(string(data), want) {
|
||||
t.Fatalf("rendered env missing %q:\n%s", want, data)
|
||||
}
|
||||
}
|
||||
|
||||
// PUBLIC_URL canonicalization is reused from the classic installer.
|
||||
data, err = (ActivationOptions{
|
||||
Activate: true, URL: "https://rsmon.ru", Token: "secret",
|
||||
PublicURL: "https://worker.example.com",
|
||||
}).renderEnv()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.Contains(string(data), "PUBLIC_URL=https://worker.example.com\n") {
|
||||
t.Fatalf("rendered env missing PUBLIC_URL:\n%s", data)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRenderEnvActivationRejectsMixedBasicAuth(t *testing.T) {
|
||||
t.Setenv("RSMON_TOKEN", "")
|
||||
if _, err := (ActivationOptions{
|
||||
Activate: true, URL: "https://rsmon.ru", Token: "secret",
|
||||
Login: "admin",
|
||||
}).renderEnv(); err == nil {
|
||||
t.Fatal("login-only basic auth accepted")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRenderEnvActivationRequiresToken(t *testing.T) {
|
||||
if _, err := (ActivationOptions{Activate: true, URL: "https://rsmon.ru"}).renderEnv(); err == nil {
|
||||
t.Fatal("activation without a token accepted")
|
||||
}
|
||||
if _, err := (ActivationOptions{Activate: true, Token: "x", URL: "not-a-url"}).renderEnv(); err == nil {
|
||||
t.Fatal("activation with a malformed URL accepted")
|
||||
}
|
||||
if _, err := (ActivationOptions{Activate: true, Token: "x", URL: "https://rsmon.ru", Name: "Bad_Name"}).renderEnv(); err == nil {
|
||||
t.Fatal("activation with an invalid instance name accepted")
|
||||
}
|
||||
if _, err := (ActivationOptions{Activate: true, Token: "x", URL: "https://rsmon.ru", Name: "edge"}).renderEnv(); err == nil {
|
||||
t.Fatal("named activation without WORKER_PORT accepted")
|
||||
}
|
||||
}
|
||||
|
||||
// TestRenderEnvActivationReadsEnvFileOnce proves the env file is read a
|
||||
// single time and validated from the in-memory bytes: swapping the file
|
||||
// after the read cannot smuggle a different value into the render, and a
|
||||
// malformed file fails on the read bytes.
|
||||
func TestRenderEnvActivationReadsEnvFileOnce(t *testing.T) {
|
||||
t.Setenv("RSMON_TOKEN", "")
|
||||
path := filepath.Join(t.TempDir(), "worker.env")
|
||||
if err := os.WriteFile(path, []byte("RSMON_URL=https://rsmon.ru\nRSMON_TOKEN=file-token\nWORKER_PORT=28888\n"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
data, err := (ActivationOptions{Activate: true, EnvFile: path}).renderEnv()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.Contains(string(data), "RSMON_TOKEN=file-token\n") || !strings.Contains(string(data), "WORKER_PORT=28888\n") {
|
||||
t.Fatalf("env file values not rendered:\n%s", data)
|
||||
}
|
||||
// A malformed file must be rejected from the same read.
|
||||
if err := os.WriteFile(path, []byte("RSMON_URL=https://rsmon.ru\nRSMON_TOKEN=has space\n"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := (ActivationOptions{Activate: true, EnvFile: path}).renderEnv(); err == nil {
|
||||
t.Fatal("malformed env file accepted")
|
||||
}
|
||||
}
|
||||
|
||||
// TestNormalizeSourceOptionsActivationRendersOnce verifies the rendered
|
||||
// env is computed during normalization and reused, so a later call cannot
|
||||
// re-read a changed env file.
|
||||
func TestNormalizeSourceOptionsActivationRendersOnce(t *testing.T) {
|
||||
t.Setenv("RSMON_TOKEN", "")
|
||||
path := filepath.Join(t.TempDir(), "worker.env")
|
||||
if err := os.WriteFile(path, []byte("RSMON_URL=https://rsmon.ru\nRSMON_TOKEN=once-token\n"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
base := SourceInstallOptions{
|
||||
SSHOptions: SSHOptions{Host: "h", User: "u"},
|
||||
Activation: ActivationOptions{Activate: true, EnvFile: path},
|
||||
}
|
||||
norm, err := normalizeSourceOptions(base)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.Contains(string(norm.activationEnv), "RSMON_TOKEN=once-token\n") {
|
||||
t.Fatalf("activation env not cached during normalization: %q", norm.activationEnv)
|
||||
}
|
||||
// Even after the file changes, the cached render is authoritative.
|
||||
if err := os.WriteFile(path, []byte("RSMON_URL=https://evil.test\nRSMON_TOKEN=evil\n"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if strings.Contains(string(norm.activationEnv), "evil") {
|
||||
t.Fatalf("changed env file leaked into the cached render: %q", norm.activationEnv)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeSourceOptionsActivation(t *testing.T) {
|
||||
t.Setenv("RSMON_TOKEN", "")
|
||||
base := SourceInstallOptions{SSHOptions: SSHOptions{Host: "h", User: "u"}}
|
||||
if _, err := normalizeSourceOptions(base); err != nil {
|
||||
t.Fatalf("staging-only options must stay valid: %v", err)
|
||||
}
|
||||
act := base
|
||||
act.Activation = ActivationOptions{Activate: true, URL: "https://rsmon.ru"}
|
||||
if _, err := normalizeSourceOptions(act); err == nil || !strings.Contains(err.Error(), "RSMON_TOKEN") {
|
||||
t.Fatalf("activation without a token error = %v", err)
|
||||
}
|
||||
ok := base
|
||||
ok.Activation = ActivationOptions{Activate: true, URL: "https://rsmon.ru", Token: "secret"}
|
||||
if _, err := normalizeSourceOptions(ok); err != nil {
|
||||
t.Fatalf("valid activation rejected: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateRenderedEnv(t *testing.T) {
|
||||
if err := validateRenderedEnv([]byte("RSMON_URL=https://rsmon.ru\nRSMON_TOKEN=secret\n")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := validateRenderedEnv([]byte("RSMON_TOKEN=secret\n")); err == nil {
|
||||
t.Fatal("env without RSMON_URL accepted")
|
||||
}
|
||||
if err := validateRenderedEnv([]byte("RSMON_URL=https://rsmon.ru\nRSMON_TOKEN=secret value\n")); err == nil {
|
||||
t.Fatal("env with whitespace accepted")
|
||||
}
|
||||
}
|
||||
@@ -50,11 +50,23 @@ type SourceInstallOptions struct {
|
||||
ToolchainDir string
|
||||
// StageBinary is where the built worker binary is written. It must
|
||||
// be absolute and defaults to <BuildDir>/rsmon-worker. The running
|
||||
// service and its config are NOT touched by this work package.
|
||||
// service and its config are NOT touched until Activation runs.
|
||||
StageBinary string
|
||||
// SessionTimeout bounds each remote command. 0 uses the default
|
||||
// (30 minutes); the build step can legitimately run for minutes.
|
||||
SessionTimeout time.Duration
|
||||
// Activation drives work package 4: after the staging build, the
|
||||
// staged binary, validated environment, data dir, and the detected
|
||||
// init's service definition are installed atomically and the worker
|
||||
// is started and verified (process + /healthz). Any failure rolls
|
||||
// back to the prior working install. Empty keeps SourceInstall at the
|
||||
// staging boundary and touches no service configuration.
|
||||
Activation ActivationOptions
|
||||
|
||||
// activationEnv is the once-rendered activation environment, computed
|
||||
// during option normalization so the env file is read and rendered
|
||||
// exactly once per run (no TOCTOU between preflight and activation).
|
||||
activationEnv []byte
|
||||
}
|
||||
|
||||
// SourceInstallResult is what a source installation resolved to. The
|
||||
@@ -70,6 +82,10 @@ type SourceInstallResult struct {
|
||||
ResolvedCommit string
|
||||
RecordFile string
|
||||
StageBinary string
|
||||
// Activation is set when the staged binary was atomically installed
|
||||
// and the worker started/verified. It records the installed layout
|
||||
// and the supervisor used. Nil when Activation was not requested.
|
||||
Activation *ActivationResult
|
||||
}
|
||||
|
||||
// defaultToolchainDir is the standard Go installation prefix.
|
||||
@@ -142,14 +158,20 @@ func (e *sourceExecutor) fileProber() sshinstall.FileProber {
|
||||
// branch, builds the worker to a staging path, and only then records the
|
||||
// resolved branch and commit.
|
||||
//
|
||||
// The running service, its configuration, and its data directory are
|
||||
// deliberately untouched: atomic activation and rollback are the next
|
||||
// work package. Every remote step runs with the same privilege path as
|
||||
// `deploy` (root, passwordless sudo, or sudo -S), every interpolated
|
||||
// value is single-quoted, every step script fails closed (`set -eu` or
|
||||
// explicit `&&`/retry), and every failure returns a bounded, actionable
|
||||
// error. Each remote command is capped by SessionTimeout and its stdout
|
||||
// is size-bounded.
|
||||
// When Activation.Activate is set, the staged binary is then atomically
|
||||
// installed together with the validated environment, data directory, and
|
||||
// the detected init's service definition, and the worker is started and
|
||||
// verified (process + /healthz); any activation/start/health failure
|
||||
// rolls back to the prior working install. Without activation the
|
||||
// running service, its configuration, and its data directory are
|
||||
// deliberately untouched (the staging boundary of work package 3).
|
||||
//
|
||||
// Every remote step runs with the same privilege path as `deploy` (root,
|
||||
// passwordless sudo, or sudo -S), every interpolated value is
|
||||
// single-quoted, every step script fails closed (`set -eu` or explicit
|
||||
// `&&`/retry), and every failure returns a bounded, actionable error.
|
||||
// Each remote command is capped by SessionTimeout and its stdout is
|
||||
// size-bounded.
|
||||
func SourceInstall(opts SourceInstallOptions) (*SourceInstallResult, error) {
|
||||
opts, err := normalizeSourceOptions(opts)
|
||||
if err != nil {
|
||||
@@ -276,6 +298,15 @@ func SourceInstall(opts SourceInstallOptions) (*SourceInstallResult, error) {
|
||||
if _, err := executor.runPrivileged("sh -c " + shellQuote(commitRecordScript(plan.BuildDir, branch, commit))); err != nil {
|
||||
return nil, fmt.Errorf("record resolved commit: %w", err)
|
||||
}
|
||||
|
||||
// Work package 4: atomically install the staged build and activate
|
||||
// the worker. On any failure the remote script restores the previous
|
||||
// working install and this step returns a bounded error.
|
||||
if opts.Activation.Activate {
|
||||
if err := executor.activateWorker(opts, result); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
@@ -316,6 +347,20 @@ func normalizeSourceOptions(o SourceInstallOptions) (SourceInstallOptions, error
|
||||
if o.SessionTimeout <= 0 {
|
||||
o.SessionTimeout = defaultSessionTimeout
|
||||
}
|
||||
if o.Activation.Activate {
|
||||
// Render (and validate) the activation environment exactly once
|
||||
// here. The rendered bytes are reused at activation time, so the
|
||||
// env file is read a single time and cannot change between the
|
||||
// preflight and the remote install (env-file TOCTOU).
|
||||
env, err := o.Activation.renderEnv()
|
||||
if err != nil {
|
||||
return o, fmt.Errorf("activation: %w", err)
|
||||
}
|
||||
if err := validateRenderedEnv(env); err != nil {
|
||||
return o, fmt.Errorf("activation: %w", err)
|
||||
}
|
||||
o.activationEnv = env
|
||||
}
|
||||
return o, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"crypto/ed25519"
|
||||
"crypto/rand"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"strings"
|
||||
"sync"
|
||||
@@ -106,7 +107,18 @@ func (s *fakeSSHServer) handleConn(conn net.Conn, config *ssh.ServerConfig) {
|
||||
}
|
||||
go func() {
|
||||
defer channel.Close()
|
||||
// Drain the client's stdin so uploads (which stream base64
|
||||
// over the session stdin) never block the channel window.
|
||||
// The channel is not closed until the client's write side is
|
||||
// exhausted, mirroring the real server behavior.
|
||||
var drained sync.WaitGroup
|
||||
drained.Add(1)
|
||||
go func() {
|
||||
defer drained.Done()
|
||||
_, _ = io.Copy(io.Discard, channel)
|
||||
}()
|
||||
s.handleSession(channel, requests)
|
||||
drained.Wait()
|
||||
}()
|
||||
}
|
||||
}
|
||||
@@ -151,9 +163,10 @@ func (s *fakeSSHServer) execCommand(channel ssh.Channel, command string) {
|
||||
|
||||
// defaultFakeExec simulates a minimal Linux host: it answers os-release,
|
||||
// uname, the init-marker file probe, origin URL, branch resolution, and
|
||||
// rev-parse, and accepts every install step. A pinned-branch existence
|
||||
// check (show-ref) fails by default so the missing-branch path is
|
||||
// exercised without extra setup.
|
||||
// rev-parse, accepts every install step, and simulates the work-package-4
|
||||
// activation (running-init probe, env/unit uploads, and the activate
|
||||
// script succeeding). A pinned-branch existence check (show-ref) fails by
|
||||
// default so the missing-branch path is exercised without extra setup.
|
||||
func defaultFakeExec(command string) (string, string, int) {
|
||||
switch {
|
||||
case strings.Contains(command, "cat /etc/os-release"):
|
||||
@@ -170,6 +183,18 @@ func defaultFakeExec(command string) (string, string, int) {
|
||||
return fakeCommitHex + "\n", "", 0
|
||||
case strings.Contains(command, "show-ref"):
|
||||
return "", "branch not found", 1
|
||||
case strings.Contains(command, "softlevel"):
|
||||
// running-init probe: no systemd/openrc is actually booted.
|
||||
return "none\n", "", 0
|
||||
case strings.Contains(command, "mktemp -d /tmp/rsmon-worker-act"):
|
||||
// server-side secure 0700 upload dir.
|
||||
return "/tmp/rsmon-worker-act-fake\n", "", 0
|
||||
case strings.Contains(command, "base64 -d"):
|
||||
// env/unit uploads succeed.
|
||||
return "", "", 0
|
||||
case strings.Contains(command, "rsmon-worker activated"):
|
||||
// the activate script completed successfully.
|
||||
return "", "", 0
|
||||
default:
|
||||
return "", "", 0
|
||||
}
|
||||
@@ -184,9 +209,25 @@ func testSSHOptions(port int) SourceInstallOptions {
|
||||
Password: fakeSSHPassword,
|
||||
InsecureHostKey: true,
|
||||
},
|
||||
// Staging-only by default so the work-package-3 orchestration
|
||||
// tests keep their historical shape; activation tests opt in.
|
||||
Activation: ActivationOptions{Activate: false},
|
||||
}
|
||||
}
|
||||
|
||||
// testActivationOptions wraps testSSHOptions with the credentials and
|
||||
// activation flag needed to run the work-package-4 flow against the fake
|
||||
// server.
|
||||
func testActivationOptions(port int) SourceInstallOptions {
|
||||
opts := testSSHOptions(port)
|
||||
opts.Activation = ActivationOptions{
|
||||
Activate: true,
|
||||
URL: "https://rsmon.ru",
|
||||
Token: fakeSSHPassword + "-token",
|
||||
}
|
||||
return opts
|
||||
}
|
||||
|
||||
func TestSourceInstallSSHFlow(t *testing.T) {
|
||||
srv := startFakeSSHServer(t, nil)
|
||||
res, err := SourceInstall(testSSHOptions(srv.Port()))
|
||||
@@ -349,3 +390,147 @@ func TestSourceInstallSSHCommandTimeout(t *testing.T) {
|
||||
t.Fatalf("err = %v, want command timeout", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestSourceInstallSSHActivationFlow verifies the work-package-4 flow
|
||||
// over a real SSH session: after the staging build and commit record, the
|
||||
// installer probes the running init, uploads the env and service
|
||||
// definition, runs the atomic activation script, and reports the installed
|
||||
// layout. The secrets-absent contract holds: the worker token never
|
||||
// reaches a remote command.
|
||||
func TestSourceInstallSSHActivationFlow(t *testing.T) {
|
||||
srv := startFakeSSHServer(t, nil)
|
||||
res, err := SourceInstall(testActivationOptions(srv.Port()))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if res.Activation == nil {
|
||||
t.Fatal("activation result missing")
|
||||
}
|
||||
if res.Activation.Binary != "/usr/local/bin/rsmon-worker" ||
|
||||
res.Activation.EnvFile != "/etc/rsmon-worker/worker.env" ||
|
||||
res.Activation.DataDir != "/var/lib/rsmon-worker" ||
|
||||
res.Activation.UnitFile != "/etc/systemd/system/rsmon-worker.service" ||
|
||||
res.Activation.Supervisor != "none" ||
|
||||
!res.Activation.Started {
|
||||
t.Fatalf("activation result = %+v", res.Activation)
|
||||
}
|
||||
|
||||
commands := srv.Commands()
|
||||
var joined strings.Builder
|
||||
for _, c := range commands {
|
||||
joined.WriteString(c)
|
||||
joined.WriteString("\n")
|
||||
}
|
||||
for _, want := range []string{
|
||||
"/run/systemd/system",
|
||||
"softlevel",
|
||||
"mktemp -d /tmp/rsmon-worker-act",
|
||||
"umask 077; base64 -d >",
|
||||
"rsmon-worker activated",
|
||||
"install -m 0755",
|
||||
"install -m 0600",
|
||||
"chmod 0750",
|
||||
"worker.pid",
|
||||
"liveness",
|
||||
"rm -rf --",
|
||||
} {
|
||||
if !strings.Contains(joined.String(), want) {
|
||||
t.Fatalf("activation commands missing %q:\n%s", want, joined.String())
|
||||
}
|
||||
}
|
||||
|
||||
// Ordering: build and commit record, then a server-side secure upload
|
||||
// dir is created, the env/unit are uploaded into it, and the activation
|
||||
// script runs last.
|
||||
idx := func(sub string) int {
|
||||
for i, c := range commands {
|
||||
if strings.Contains(c, sub) {
|
||||
return i
|
||||
}
|
||||
}
|
||||
t.Fatalf("command %q not found in %v", sub, commands)
|
||||
return -1
|
||||
}
|
||||
if idx("branch=%s") >= idx("mktemp -d /tmp/rsmon-worker-act") ||
|
||||
idx("mktemp -d /tmp/rsmon-worker-act") >= idx("base64 -d >") ||
|
||||
idx("base64 -d >") >= idx("rsmon-worker activated") {
|
||||
t.Fatalf("activation order wrong: %v", commands)
|
||||
}
|
||||
if strings.Contains(joined.String(), fakeSSHPassword+"-token") {
|
||||
t.Fatal("worker token leaked into a remote command")
|
||||
}
|
||||
}
|
||||
|
||||
// TestSourceInstallSSHNoActivationSkipsService proves the staging
|
||||
// boundary: without Activation the installer never probes the init system,
|
||||
// uploads an env/unit, or runs the activation script.
|
||||
func TestSourceInstallSSHNoActivationSkipsService(t *testing.T) {
|
||||
srv := startFakeSSHServer(t, nil)
|
||||
res, err := SourceInstall(testSSHOptions(srv.Port()))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if res.Activation != nil {
|
||||
t.Fatalf("activation must not run without the flag: %+v", res.Activation)
|
||||
}
|
||||
joined := strings.Join(srv.Commands(), "\n")
|
||||
for _, forbidden := range []string{"softlevel", "base64 -d >", "rsmon-worker activated", "worker.pid"} {
|
||||
if strings.Contains(joined, forbidden) {
|
||||
t.Fatalf("staging-only flow ran activation step %q:\n%s", forbidden, joined)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestSourceInstallSSHActivationRequiresToken verifies the credentials
|
||||
// gate: activation without a token fails before any remote connection.
|
||||
func TestSourceInstallSSHActivationRequiresToken(t *testing.T) {
|
||||
srv := startFakeSSHServer(t, nil)
|
||||
opts := testSSHOptions(srv.Port())
|
||||
opts.Activation = ActivationOptions{Activate: true, URL: "https://rsmon.ru"}
|
||||
_, err := SourceInstall(opts)
|
||||
if err == nil || !strings.Contains(err.Error(), "RSMON_TOKEN") {
|
||||
t.Fatalf("err = %v, want token requirement", err)
|
||||
}
|
||||
if got := len(srv.Commands()); got != 0 {
|
||||
t.Fatalf("commands ran before validation failed: %d", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestSourceInstallSSHActivationFailure verifies the bounded failure
|
||||
// path: a failing activate script surfaces as a labelled error with the
|
||||
// remote stderr, and the uploaded temp env/unit are cleaned up by the
|
||||
// controller defer regardless of the outcome.
|
||||
func TestSourceInstallSSHActivationFailure(t *testing.T) {
|
||||
srv := startFakeSSHServer(t, func(command string) (string, string, int) {
|
||||
if strings.Contains(command, "rsmon-worker activated") {
|
||||
return "", "activation exploded: disk full", 1
|
||||
}
|
||||
return defaultFakeExec(command)
|
||||
})
|
||||
_, err := SourceInstall(testActivationOptions(srv.Port()))
|
||||
if err == nil || !strings.Contains(err.Error(), "activate worker service") || !strings.Contains(err.Error(), "disk full") {
|
||||
t.Fatalf("err = %v, want bounded activation failure", err)
|
||||
}
|
||||
joined := strings.Join(srv.Commands(), "\n")
|
||||
if !strings.Contains(joined, "rm -rf --") {
|
||||
t.Fatalf("secure upload dir cleanup not issued after activation failure:\n%s", joined)
|
||||
}
|
||||
}
|
||||
|
||||
// TestSourceInstallSSHActivationHealthFailure verifies the health gate
|
||||
// surfaces as an activation failure and the temp files are still cleaned.
|
||||
// The fake cannot run the shell script, so the failure is injected at the
|
||||
// activate-command level; the real rollback semantics are covered by the
|
||||
// Docker/OpenSSH E2E tests.
|
||||
func TestSourceInstallSSHActivationHealthFailure(t *testing.T) {
|
||||
srv := startFakeSSHServer(t, func(command string) (string, string, int) {
|
||||
if strings.Contains(command, "rsmon-worker activated") {
|
||||
return "", "health failure: worker did not answer /healthz", 1
|
||||
}
|
||||
return defaultFakeExec(command)
|
||||
})
|
||||
_, err := SourceInstall(testActivationOptions(srv.Port()))
|
||||
if err == nil || !strings.Contains(err.Error(), "did not answer /healthz") {
|
||||
t.Fatalf("err = %v, want /healthz verification failure", err)
|
||||
}
|
||||
}
|
||||
|
||||
Ссылка в новой задаче
Block a user