feat: publish standalone worker

Separate worker packaging and service lifecycle from the control plane.
2026-07-13 17:55:14 +03:00
--- a/cmd/rsmon-worker/main.go
+++ b/cmd/rsmon-worker/main.go
@@ -0,0 +1,588 @@
+// Distributed monitoring worker binary.
+package main
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"log"
+	"net"
+	"net/http"
+	"os"
+	"os/signal"
+	"strconv"
+	"strings"
+	"syscall"
+	"time"
+
+	"github.com/hashicorp/raft"
+	"github.com/joho/godotenv"
+
+	"rsgit.ru/rsmon/rsmon/internal/distworker"
+	"rsgit.ru/rsmon/rsmon/internal/webapp"
+	"rsgit.ru/rsmon/rsmon/internal/workercluster"
+)
+
+var (
+	// Build info set by ldflags.
+	version   = "dev"
+	commit    = "unknown"
+	buildDate = "unknown"
+
+	// webappEnabled flips the local web UI on at boot. Default true.
+	// Phase 1 keeps it on; the flag exists so a Phase 2 basic-auth
+	// install can opt out without recompiling.
+	webappEnabled = true
+
+	// clusterDebugApplyTestConfig, when true, submits the hardcoded
+	// CriticalCheckConfig from workercluster.DefaultDebugCriticalCheck
+	// to the cluster on startup. Wired via the
+	// --cluster-debug-apply-test-config CLI flag; the e2e script
+	// uses this so it can verify FSM replication without the
+	// signed-config-adoption producer (which lands in a later phase).
+	//
+	// DEBUG: this flag is a placeholder. It must be removed (or
+	// guarded behind a build tag) before any production build.
+	//
+	// TODO(phase-N): remove once the real producer is wired.
+	clusterDebugApplyTestConfig = false
+)
+
+func main() {
+	log.SetFlags(log.LstdFlags | log.Lshortfile)
+	loadDotEnv()
+
+	versionFlag := flag.Bool("version", false, "Print version and exit")
+	noWeb := flag.Bool("no-web", false, "Disable the local web UI (Phase 1 ships with it on)")
+	debugApplyConfig := flag.Bool("cluster-debug-apply-test-config", false,
+		"Submit a hardcoded CriticalCheckConfig to the cluster on startup. DEBUG: remove once the real config.adopt producer is wired.")
+	flag.Parse()
+
+	if *versionFlag {
+		fmt.Printf("rsmon-worker version=%s commit=%s buildDate=%s\n", version, commit, buildDate)
+		os.Exit(0)
+	}
+	webappEnabled = !*noWeb
+	clusterDebugApplyTestConfig = *debugApplyConfig
+
+	if len(flag.Args()) > 0 && flag.Arg(0) == "health" {
+		os.Exit(healthCheck())
+	}
+
+	log.Println("rsmon-worker starting...")
+
+	cfg := distworker.ConfigFromEnv()
+	logHTTPSettings(cfg.HTTP, webappEnabled)
+	// The HTTP listener is started by the webapp. WORKER_LOGIN /
+	// WORKER_PASSWORD (basic auth) are passed through to the webapp
+	// Config; the webapp's ValidateBasicAuth rejects XOR.
+	if err := webapp.ValidateBasicAuth(cfg.HTTP.Login, cfg.HTTP.Password); err != nil {
+		log.Fatalf("worker: %v", err)
+	}
+
+	runner := distworker.NewRunner(&cfg)
+
+	// Graceful shutdown context shared by the runner, the cluster,
+	// and the webapp. ctxCancel is called by the signal handler so
+	// all three wind down together; the runner waits for its
+	// goroutines, the cluster drains its rafthttp listener + raft
+	// state machine, then the webapp closes its listener and the
+	// SQLite handle. The defer is a safety net for early returns
+	// before the signal handler registers (the handler always wins
+	// for SIGINT/SIGTERM, but other early exits rely on the defer).
+	ctx, ctxCancel := context.WithCancel(context.Background())
+	defer func() { ctxCancel() }() //nolint:gocritic // safety net for early returns; signal handler owns the canonical path
+
+	sigCh := make(chan os.Signal, 1)
+	signal.Notify(sigCh, syscall.SIGINT, syscall.SIGTERM)
+	go func() {
+		<-sigCh
+		log.Println("worker: shutdown signal received")
+		runner.Stop()
+		ctxCancel()
+	}()
+
+	// Construct the cluster subsystem first (Task 5) so its admin
+	// endpoints can be wired into the webapp. The cluster is optional;
+	// when WORKER_CLUSTER_ENABLED=false (the default) we skip it and
+	// the webapp falls back to the local-only auth path.
+	cluster, clusterView, err := buildCluster(ctx, &cfg)
+	if err != nil {
+		// buildCluster never returns a partial cluster on error, so
+		// nothing to clean up here. Print and exit so the trailing
+		// defer (which only runs when cluster != nil) does not
+		// confuse linters or runtime observers.
+		log.Printf("worker: cluster init failed: %v", err)
+		os.Exit(1) //nolint:gocritic // safety net; see comment above
+	}
+	if cluster != nil {
+		defer func() {
+			shut, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+			defer cancel()
+			if err := cluster.Shutdown(shut); err != nil {
+				log.Printf("worker: cluster shutdown: %v", err)
+			}
+		}()
+	}
+
+	// Run the runner and the webapp concurrently. The runner blocks
+	// until ctxCancel; the webapp is started in its own goroutine
+	// so a web-side error does not block the runner. Both unwind
+	// when ctxCancel fires.
+	webappErrCh := make(chan error, 1)
+	if webappEnabled {
+		srv, _, err := buildWebapp(ctx, runner, clusterView)
+		if err != nil {
+			// No deferred cleanup needed: the runner has not been
+			// started yet, the signal handler has not registered,
+			// and the only shared resource is the context which
+			// has nothing tied to it. log.Fatalf calls os.Exit so
+			// the deferred ctxCancel would be redundant noise.
+			log.Fatalf("worker: webapp init failed: %v", err)
+		}
+		go func() {
+			webappErrCh <- srv.Start(ctx)
+		}()
+		// Provision the first-run user before the runner gets a
+		// chance to send its first websocket hello so the operator
+		// can log in immediately if the main app is slow to ack.
+		if err := webapp.ProvisionFirstRunIfNeeded(srv, log.New(os.Stderr, "webapp: ", log.LstdFlags)); err != nil {
+			log.Printf("worker: webapp first-run provisioning: %v", err)
+		}
+	}
+
+	if err := runner.Start(); err != nil {
+		log.Fatal("worker failed:", err)
+	}
+
+	// Runner returned: the signal handler already canceled ctx so
+	// the webapp goroutine will exit shortly. Wait for it to avoid
+	// leaking the SQLite handle.
+	if webappEnabled {
+		select {
+		case err := <-webappErrCh:
+			if err != nil {
+				log.Printf("worker: webapp exited: %v", err)
+			}
+		case <-time.After(5 * time.Second):
+			log.Printf("worker: webapp shutdown timed out")
+		}
+	}
+}
+
+// buildWebapp wires the worker view into a webapp.Server. The Deps
+// adapter reads from the runner (which is not yet Started at the
+// time this is called; recent buffers are empty by design).
+func buildWebapp(_ context.Context, runner *distworker.Runner, cluster webapp.ClusterView) (*webapp.Server, *webapp.Deps, error) {
+	deps := &webapp.Deps{
+		Runner:    runnerWrapper{runner},
+		Cluster:   cluster,
+		Version:   version,
+		BuildDate: buildDate,
+		Commit:    commit,
+		StartedAt: time.Now().UTC(),
+		Logger:    log.New(os.Stderr, "webapp: ", log.LstdFlags|log.Lshortfile),
+		TokenRotator: func(ctx context.Context) (string, error) {
+			return runner.RotateToken(ctx)
+		},
+		ReleaseHTTPClient: &http.Client{
+			Timeout: 5 * time.Second,
+			Transport: &http.Transport{
+				Proxy: http.ProxyFromEnvironment,
+			},
+		},
+	}
+	srv, err := webapp.New(webapp.ConfigFromEnvOrDefault(), deps)
+	if err != nil {
+		return nil, deps, err
+	}
+	return srv, deps, nil
+}
+
+// buildCluster reads WORKER_CLUSTER_* env vars and, when cluster mode
+// is enabled, constructs and starts a *workercluster.Cluster. The
+// returned ClusterView is the narrow interface webapp consumes; it is
+// nil when the cluster is not enabled.
+//
+// The cluster's rafthttp listener binds to 127.0.0.1 on
+// WORKER_CLUSTER_PORT (default = WORKER_PORT + 10000) so the
+// worker webapp listener and the raft transport do not collide. The
+// peers list (WORKER_CLUSTER_PEERS) is parsed as a comma-separated
+// list of "nodeID@host:port" entries; the first peer becomes the
+// Seed for non-bootstrap nodes.
+//
+// The function never returns a partial cluster: either both the
+// concrete *Cluster and the ClusterView are returned, or both are nil
+// (cluster disabled) or an error is returned (cluster enabled but
+// misconfigured).
+func buildCluster(ctx context.Context, cfg *distworker.Config) (*workercluster.Cluster, webapp.ClusterView, error) {
+	if !clusterModeEnabled() {
+		return nil, nil, nil
+	}
+	creds := workercluster.HTTPCreds{Login: cfg.HTTP.Login, Password: cfg.HTTP.Password}
+	if !creds.IsConfigured() {
+		return nil, nil, fmt.Errorf(
+			"worker: WORKER_CLUSTER_ENABLED=true requires WORKER_LOGIN and WORKER_PASSWORD (rafthttp basic auth)")
+	}
+
+	nodeID := strings.TrimSpace(os.Getenv("WORKER_CLUSTER_ID"))
+	if nodeID == "" {
+		return nil, nil, fmt.Errorf("worker: WORKER_CLUSTER_ID is required when WORKER_CLUSTER_ENABLED=true")
+	}
+	dataDir := strings.TrimSpace(os.Getenv("WORKER_CLUSTER_DATA_DIR"))
+	if dataDir == "" {
+		return nil, nil, fmt.Errorf("worker: WORKER_CLUSTER_DATA_DIR is required when WORKER_CLUSTER_ENABLED=true")
+	}
+	if err := workercluster.EnsureDataDir(dataDir); err != nil {
+		return nil, nil, fmt.Errorf("worker: WORKER_CLUSTER_DATA_DIR %q: %w", dataDir, err)
+	}
+
+	port := clusterPort()
+	host := strings.TrimSpace(os.Getenv("WORKER_CLUSTER_HOST"))
+	if host == "" {
+		host = "127.0.0.1"
+	}
+	localAddr := net.JoinHostPort(host, port)
+
+	peers, err := parseClusterPeers(os.Getenv("WORKER_CLUSTER_PEERS"))
+	if err != nil {
+		return nil, nil, fmt.Errorf("worker: WORKER_CLUSTER_PEERS: %w", err)
+	}
+
+	opts := &workercluster.Options{
+		NodeID:           nodeID,
+		LocalAddr:        localAddr,
+		DataDir:          dataDir,
+		Creds:            creds,
+		HeartbeatTimeout: 1000 * time.Millisecond,
+		ElectionTimeout:  3000 * time.Millisecond,
+		Logger:           log.New(os.Stderr, "[workercluster] ", log.LstdFlags),
+		LogOutput:        os.Stderr,
+	}
+	// WORKER_CLUSTER_BOOTSTRAP=true forces bootstrap mode even
+	// when WORKER_CLUSTER_PEERS is set (the peers list is then
+	// informational; the cluster subsystem records it but does
+	// not dial). Without it, a non-empty peers list means the
+	// node joins via the first peer. An empty peers list always
+	// bootstraps.
+	bootstrap := !hasSeedPeer(peers)
+	if v := strings.ToLower(strings.TrimSpace(os.Getenv("WORKER_CLUSTER_BOOTSTRAP"))); v == "true" || v == "1" || v == "yes" {
+		bootstrap = true
+	}
+	if bootstrap {
+		opts.Bootstrap = true
+	} else {
+		opts.Seed = peers[0]
+	}
+
+	c, err := workercluster.New(opts)
+	if err != nil {
+		return nil, nil, fmt.Errorf("worker: cluster.New: %w", err)
+	}
+	if err := c.Start(ctx); err != nil {
+		return nil, nil, fmt.Errorf("worker: cluster.Start: %w", err)
+	}
+	log.Printf("worker cluster: started node_id=%s addr=%s bootstrap=%t peers=%d",
+		nodeID, localAddr, bootstrap, len(peers))
+
+	if clusterDebugApplyTestConfig && bootstrap {
+		// The debug apply only fires on bootstrap nodes; a
+		// joiner cannot commit a log entry until it has been
+		// promoted to voter. Wait for this node to win an
+		// election first (a fresh single-voter cluster elects
+		// itself immediately but the goroutine may run before
+		// the state has flipped).
+		go func() {
+			deadline := time.Now().Add(15 * time.Second)
+			for time.Now().Before(deadline) {
+				if c.Raft() != nil && c.Raft().State() == raft.Leader {
+					break
+				}
+				time.Sleep(100 * time.Millisecond)
+			}
+			if c.Raft() == nil || c.Raft().State() != raft.Leader {
+				log.Printf("worker cluster: applying debug test config id=%d: not leader after 15s",
+					workercluster.DefaultDebugCriticalCheck().ID)
+				return
+			}
+			check := workercluster.DefaultDebugCriticalCheck()
+			applied, err := c.ApplyTestConfig(&check)
+			if err != nil {
+				log.Printf("worker cluster: applying debug test config id=%d: %v",
+					check.ID, err)
+				return
+			}
+			log.Printf("worker cluster: applying debug test config id=%d applied_index=%d",
+				check.ID, applied)
+		}()
+	}
+
+	return c, &clusterAdapter{c: c}, nil
+}
+
+// clusterAdapter wraps *workercluster.Cluster so it implements the
+// webapp.ClusterView interface without webapp importing the raft
+// code path. The ApplyTestConfig signature is the one webapp expects
+// (no config argument; the cluster subsystem owns the hardcoded
+// payload so the two sides cannot drift).
+type clusterAdapter struct {
+	c *workercluster.Cluster
+}
+
+func (a *clusterAdapter) Stats() webapp.ClusterStats {
+	src := a.c.ClusterStats()
+	return webapp.ClusterStats{
+		NodeID:           src.NodeID,
+		LocalAddr:        src.LocalAddr,
+		State:            src.State,
+		Leader:           src.Leader,
+		Term:             src.Term,
+		AppliedIndex:     src.AppliedIndex,
+		LastIndex:        src.LastIndex,
+		NumPeers:         src.NumPeers,
+		Voters:           src.Voters,
+		FSMChecks:        src.FSMChecks,
+		FSMMembers:       src.FSMMembers,
+		FSMConfigVersion: src.FSMConfigVersion,
+		FSMOutboxLen:     src.FSMOutboxLen,
+		FSMPartition:     src.FSMPartition,
+	}
+}
+
+func (a *clusterAdapter) ApplyTestConfig() (uint64, error) {
+	check := workercluster.DefaultDebugCriticalCheck()
+	return a.c.ApplyTestConfig(&check)
+}
+
+func (a *clusterAdapter) ClusterID() string { return a.c.ClusterID() }
+func (a *clusterAdapter) LocalAddr() string { return a.c.LocalAddr() }
+
+// clusterModeEnabled returns true when WORKER_CLUSTER_ENABLED is set
+// to a truthy value. Kept as a free function (not a method) so the
+// webapp's ClusterEnabledFromEnv and the cmd binary agree on the
+// parsing rules.
+func clusterModeEnabled() bool {
+	v := strings.ToLower(strings.TrimSpace(os.Getenv("WORKER_CLUSTER_ENABLED")))
+	return v == "true" || v == "1" || v == "yes"
+}
+
+// clusterPort derives the rafthttp bind port from WORKER_CLUSTER_PORT
+// or, when that env var is unset, WORKER_PORT+10000. The +10000 offset
+// keeps the webapp and the raft transport from colliding on the same
+// loopback bind.
+func clusterPort() string {
+	if raw := strings.TrimSpace(os.Getenv("WORKER_CLUSTER_PORT")); raw != "" {
+		return raw
+	}
+	base := distworker.DefaultHTTPPort
+	if raw := strings.TrimSpace(os.Getenv("WORKER_PORT")); raw != "" {
+		if v, err := strconv.Atoi(raw); err == nil && v > 0 && v <= 65535 {
+			base = v
+		}
+	}
+	return strconv.Itoa(base + 10000)
+}
+
+// parseClusterPeers parses a comma-separated list of "nodeID@host:port"
+// entries. Empty input returns an empty slice. Whitespace around entries
+// is trimmed; blank entries are rejected.
+func parseClusterPeers(raw string) ([]workercluster.Peer, error) {
+	raw = strings.TrimSpace(raw)
+	if raw == "" {
+		return nil, nil
+	}
+	var out []workercluster.Peer
+	for _, entry := range strings.Split(raw, ",") {
+		entry = strings.TrimSpace(entry)
+		if entry == "" {
+			return nil, fmt.Errorf("empty peer entry in %q", raw)
+		}
+		at := strings.LastIndex(entry, "@")
+		if at < 0 {
+			return nil, fmt.Errorf("peer entry %q missing '@' separator (expected nodeID@host:port)", entry)
+		}
+		nodeID := strings.TrimSpace(entry[:at])
+		addr := strings.TrimSpace(entry[at+1:])
+		// Tolerate a scheme prefix; rafthttp is plain http.
+		if i := strings.Index(addr, "://"); i >= 0 {
+			addr = addr[i+3:]
+		}
+		if nodeID == "" || addr == "" {
+			return nil, fmt.Errorf("peer entry %q has empty nodeID or address", entry)
+		}
+		if _, _, err := net.SplitHostPort(addr); err != nil {
+			return nil, fmt.Errorf("peer entry %q: bad host:port: %w", entry, err)
+		}
+		out = append(out, workercluster.Peer{WorkerID: nodeID, Address: addr})
+	}
+	return out, nil
+}
+
+// hasSeedPeer reports whether the peers list contains at least one
+// usable entry. Bootstrap nodes (the first node of a new cluster) have
+// an empty peers list.
+func hasSeedPeer(peers []workercluster.Peer) bool {
+	return len(peers) > 0
+}
+
+// runnerWrapper adapts *distworker.Runner to webapp.WorkerView. Kept
+// here (not in the webapp package) so the distworker -> webapp edge
+// is owned by the binary that links both packages.
+type runnerWrapper struct {
+	r *distworker.Runner
+}
+
+func (w runnerWrapper) HTTPConfig() (cfg distworker.HTTPConfig) {
+	if w.r == nil {
+		return cfg
+	}
+	return w.r.HTTPConfig()
+}
+
+func (w runnerWrapper) Token() string {
+	if w.r == nil {
+		return ""
+	}
+	return w.r.Token()
+}
+
+func (w runnerWrapper) TokenRotatedAt() time.Time {
+	if w.r == nil {
+		return time.Time{}
+	}
+	return w.r.TokenRotatedAt()
+}
+
+func (w runnerWrapper) WorkerID() string {
+	if w.r == nil {
+		return ""
+	}
+	return w.r.WorkerID()
+}
+
+func (w runnerWrapper) RegionCode() string {
+	if w.r == nil {
+		return ""
+	}
+	return w.r.RegionCode()
+}
+
+func (w runnerWrapper) WorkerVersion() string {
+	if w.r == nil {
+		return ""
+	}
+	return w.r.WorkerVersion()
+}
+
+func (w runnerWrapper) WorkerCapabilities() []string {
+	if w.r == nil {
+		return nil
+	}
+	return w.r.WorkerCapabilities()
+}
+
+func (w runnerWrapper) LastHeartbeatAck() time.Time {
+	if w.r == nil {
+		return time.Time{}
+	}
+	return w.r.LastHeartbeatAck()
+}
+
+func (w runnerWrapper) MasterStatus() (*bool, time.Time) {
+	if w.r == nil {
+		return nil, time.Time{}
+	}
+	return w.r.MasterStatus()
+}
+
+func (w runnerWrapper) RecentResults(n int) []webapp.ResultRow {
+	src := w.r.RecentResults(n)
+	out := make([]webapp.ResultRow, len(src))
+	for i, r := range src {
+		out[i] = webapp.ResultRow{
+			MonitorID:  r.MonitorID,
+			CheckID:    r.CheckID,
+			Kind:       r.Kind,
+			Host:       r.Host,
+			State:      r.State,
+			DurationMs: r.DurationMs,
+			Error:      r.Error,
+			At:         r.At,
+		}
+	}
+	return out
+}
+
+func (w runnerWrapper) RecentNotifications(n int) []webapp.NotificationRow {
+	src := w.r.RecentNotifications(n)
+	out := make([]webapp.NotificationRow, len(src))
+	for i, r := range src {
+		out[i] = webapp.NotificationRow{
+			Kind:    r.Kind,
+			Channel: r.Channel,
+			Subject: r.Subject,
+			Body:    r.Body,
+			OK:      r.OK,
+			Error:   r.Error,
+			At:      r.At,
+		}
+	}
+	return out
+}
+
+// logHTTPSettings prints a single line summarizing the HTTP listener
+// settings the operator configured, so misconfigurations are visible at
+// startup. Login is masked. willListen flips to true once the HTTP
+// listener is actually bound (Task 3) so the same helper can be reused.
+func logHTTPSettings(h distworker.HTTPConfig, willListen bool) {
+	login := "***"
+	if h.Login == "" {
+		login = "(empty)"
+	}
+	url := h.URL
+	if url == "" {
+		url = "(empty)"
+	}
+	log.Printf("worker http settings: host=%s port=%d url=%s login=%s will_listen=%t",
+		h.Host, h.Port, url, login, willListen)
+	if h.URL != "" {
+		if host, warn := distworker.WarnInsecurePublicURL(h.URL); warn {
+			log.Printf(
+				"worker http settings: WARN WORKER_URL=http://%s uses plain HTTP on a non-loopback host; "+
+					"production deployments usually terminate TLS at a reverse proxy", host,
+			)
+		}
+	}
+}
+
+func loadDotEnv() {
+	if err := godotenv.Load(".env"); err == nil {
+		log.Println("worker .env file loaded")
+	}
+}
+
+func healthCheck() int {
+	endpoint := os.Getenv("RSMON_URL")
+	if endpoint == "" {
+		endpoint = "https://rsmon.ru"
+	}
+	endpoint = strings.TrimRight(endpoint, "/")
+	if strings.HasSuffix(endpoint, "/api/worker") {
+		endpoint = strings.TrimSuffix(endpoint, "/api/worker")
+	} else if strings.HasSuffix(endpoint, "/worker") {
+		endpoint = strings.TrimSuffix(endpoint, "/worker")
+	}
+	client := &http.Client{Timeout: 10 * time.Second}
+	resp, err := client.Get(endpoint + "/up")
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "health check failed: %v\n", err)
+		return 1
+	}
+	defer resp.Body.Close() //nolint:errcheck
+	if resp.StatusCode != http.StatusOK {
+		fmt.Fprintf(os.Stderr, "health check failed: status %s\n", resp.Status)
+		return 1
+	}
+	fmt.Println("health check ok")
+	return 0
+}