// Distributed monitoring worker binary. package main import ( "context" "flag" "fmt" "log" "net" "net/http" "os" "os/signal" "strconv" "strings" "syscall" "time" "github.com/joho/godotenv" "rocketgit.ru/rsmon/worker/internal/distworker" "rocketgit.ru/rsmon/worker/internal/webapp" "rocketgit.ru/rsmon/worker/internal/workercluster" ) var ( // Build info set by ldflags. version = "dev" commit = "unknown" buildDate = "unknown" // webappEnabled flips the local web UI on at boot. Default true. // Phase 1 keeps it on; the flag exists so a Phase 2 basic-auth // install can opt out without recompiling. webappEnabled = true ) func main() { log.SetFlags(log.LstdFlags | log.Lshortfile) // Load .env before any subcommand dispatch so management commands // (install/deploy) read the same environment as the worker runtime. // godotenv.Load only fills variables not already present in the // process environment, so an explicit export or sudo -E still wins. loadDotEnv() if handled, code := dispatchManagementCommand(os.Args[1:]); handled { os.Exit(code) } versionFlag := flag.Bool("version", false, "Print version and exit") noWeb := flag.Bool("no-web", false, "Disable the local web UI (Phase 1 ships with it on)") flag.Parse() if *versionFlag { fmt.Printf("rsmon-worker version=%s commit=%s buildDate=%s\n", version, commit, buildDate) os.Exit(0) } webappEnabled = !*noWeb if len(flag.Args()) > 0 && flag.Arg(0) == "health" { os.Exit(healthCheck()) } if len(flag.Args()) > 0 && flag.Arg(0) == "liveness" { os.Exit(livenessCheck()) } if len(flag.Args()) > 0 { fmt.Fprintf(os.Stderr, "unknown command %q\n", flag.Arg(0)) os.Exit(2) } log.Println("rsmon-worker starting...") cfg := distworker.ConfigFromEnv() logHTTPSettings(cfg.HTTP, webappEnabled) // The HTTP listener is started by the webapp. WORKER_LOGIN / // WORKER_PASSWORD (basic auth) are passed through to the webapp // Config; the webapp's ValidateBasicAuth rejects XOR. if err := webapp.ValidateBasicAuth(cfg.HTTP.Login, cfg.HTTP.Password); err != nil { log.Fatalf("worker: %v", err) } // Enforce the advertised PUBLIC_URL origin rules (and the // production HTTPS policy) before accepting work. willListen=false // because the webapp's both-empty local bcrypt mode is legitimate // and already gated by ValidateBasicAuth above; here we only check // the URL invariants. if err := distworker.ValidateHTTPConfig(cfg.HTTP, false); err != nil { log.Fatalf("worker: %v", err) } runner := distworker.NewRunner(&cfg) // Graceful shutdown context shared by the runner, the cluster, // and the webapp. ctxCancel is called by the signal handler so // all three wind down together; the runner waits for its // goroutines, the cluster drains its rafthttp listener + raft // state machine, then the webapp closes its listener and the // SQLite handle. The defer is a safety net for early returns // before the signal handler registers (the handler always wins // for SIGINT/SIGTERM, but other early exits rely on the defer). ctx, ctxCancel := context.WithCancel(context.Background()) defer func() { ctxCancel() }() //nolint:gocritic // safety net for early returns; signal handler owns the canonical path sigCh := make(chan os.Signal, 1) signal.Notify(sigCh, syscall.SIGINT, syscall.SIGTERM) go func() { <-sigCh log.Println("worker: shutdown signal received") runner.Stop() ctxCancel() }() // Construct the cluster subsystem first (Task 5) so its admin // endpoints can be wired into the webapp. The cluster is optional; // when WORKER_CLUSTER_ENABLED=false (the default) we skip it and // the webapp falls back to the local-only auth path. cluster, clusterView, err := buildCluster(ctx, &cfg) if err != nil { // buildCluster never returns a partial cluster on error, so // nothing to clean up here. Print and exit so the trailing // defer (which only runs when cluster != nil) does not // confuse linters or runtime observers. log.Printf("worker: cluster init failed: %v", err) os.Exit(1) //nolint:gocritic // safety net; see comment above } if cluster != nil { defer func() { shut, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() if err := cluster.Shutdown(shut); err != nil { log.Printf("worker: cluster shutdown: %v", err) } }() } // Run the runner and the webapp concurrently. The runner blocks // until ctxCancel; the webapp is started in its own goroutine // so a web-side error does not block the runner. Both unwind // when ctxCancel fires. webappErrCh := make(chan error, 1) if webappEnabled { srv, _, err := buildWebapp(ctx, runner, clusterView) if err != nil { // No deferred cleanup needed: the runner has not been // started yet, the signal handler has not registered, // and the only shared resource is the context which // has nothing tied to it. log.Fatalf calls os.Exit so // the deferred ctxCancel would be redundant noise. log.Fatalf("worker: webapp init failed: %v", err) } go func() { webappErrCh <- srv.Start(ctx) }() // Provision the first-run user before the runner gets a // chance to send its first websocket hello so the operator // can log in immediately if the main app is slow to ack. if err := webapp.ProvisionFirstRunIfNeeded(srv, log.New(os.Stderr, "webapp: ", log.LstdFlags)); err != nil { log.Printf("worker: webapp first-run provisioning: %v", err) } } if err := runner.Start(); err != nil { log.Fatal("worker failed:", err) } // Runner returned: the signal handler already canceled ctx so // the webapp goroutine will exit shortly. Wait for it to avoid // leaking the SQLite handle. if webappEnabled { select { case err := <-webappErrCh: if err != nil { log.Printf("worker: webapp exited: %v", err) } case <-time.After(5 * time.Second): log.Printf("worker: webapp shutdown timed out") } } } // buildWebapp wires the worker view into a webapp.Server. The Deps // adapter reads from the runner (which is not yet Started at the // time this is called; recent buffers are empty by design). func buildWebapp(_ context.Context, runner *distworker.Runner, cluster webapp.ClusterView) (*webapp.Server, *webapp.Deps, error) { deps := &webapp.Deps{ Runner: runnerWrapper{runner}, Cluster: cluster, Version: version, BuildDate: buildDate, Commit: commit, StartedAt: time.Now().UTC(), Logger: log.New(os.Stderr, "webapp: ", log.LstdFlags|log.Lshortfile), TokenRotator: func(ctx context.Context) (string, error) { return runner.RotateToken(ctx) }, ReleaseHTTPClient: &http.Client{ Timeout: 5 * time.Second, Transport: &http.Transport{ Proxy: http.ProxyFromEnvironment, }, }, } srv, err := webapp.New(webapp.ConfigFromEnvOrDefault(), deps) if err != nil { return nil, deps, err } return srv, deps, nil } // buildCluster reads WORKER_CLUSTER_* env vars and, when cluster mode // is enabled, constructs and starts a *workercluster.Cluster. The // returned ClusterView is the narrow interface webapp consumes; it is // nil when the cluster is not enabled. // // The cluster's rafthttp listener binds to 127.0.0.1 on // WORKER_CLUSTER_PORT (default = WORKER_PORT + 10000) so the // worker webapp listener and the raft transport do not collide. The // peers list (WORKER_CLUSTER_PEERS) is parsed as a comma-separated // list of "nodeID@host:port" entries; the first peer becomes the // Seed for non-bootstrap nodes. // // The function never returns a partial cluster: either both the // concrete *Cluster and the ClusterView are returned, or both are nil // (cluster disabled) or an error is returned (cluster enabled but // misconfigured). func buildCluster(ctx context.Context, cfg *distworker.Config) (*workercluster.Cluster, webapp.ClusterView, error) { if !clusterModeEnabled() { return nil, nil, nil } creds := workercluster.HTTPCreds{Login: cfg.HTTP.Login, Password: cfg.HTTP.Password} if !creds.IsConfigured() { return nil, nil, fmt.Errorf( "worker: WORKER_CLUSTER_ENABLED=true requires WORKER_LOGIN and WORKER_PASSWORD (rafthttp basic auth)") } nodeID := strings.TrimSpace(os.Getenv("WORKER_CLUSTER_ID")) if nodeID == "" { return nil, nil, fmt.Errorf("worker: WORKER_CLUSTER_ID is required when WORKER_CLUSTER_ENABLED=true") } dataDir := strings.TrimSpace(os.Getenv("WORKER_CLUSTER_DATA_DIR")) if dataDir == "" { return nil, nil, fmt.Errorf("worker: WORKER_CLUSTER_DATA_DIR is required when WORKER_CLUSTER_ENABLED=true") } if err := workercluster.EnsureDataDir(dataDir); err != nil { return nil, nil, fmt.Errorf("worker: WORKER_CLUSTER_DATA_DIR %q: %w", dataDir, err) } port := clusterPort() host := strings.TrimSpace(os.Getenv("WORKER_CLUSTER_HOST")) if host == "" { host = "127.0.0.1" } localAddr := net.JoinHostPort(host, port) peers, err := parseClusterPeers(os.Getenv("WORKER_CLUSTER_PEERS")) if err != nil { return nil, nil, fmt.Errorf("worker: WORKER_CLUSTER_PEERS: %w", err) } opts := &workercluster.Options{ NodeID: nodeID, LocalAddr: localAddr, DataDir: dataDir, Creds: creds, HeartbeatTimeout: 1000 * time.Millisecond, ElectionTimeout: 3000 * time.Millisecond, Logger: log.New(os.Stderr, "[workercluster] ", log.LstdFlags), LogOutput: os.Stderr, } // WORKER_CLUSTER_BOOTSTRAP=true forces bootstrap mode even // when WORKER_CLUSTER_PEERS is set (the peers list is then // informational; the cluster subsystem records it but does // not dial). Without it, a non-empty peers list means the // node joins via the first peer. An empty peers list always // bootstraps. bootstrap := !hasSeedPeer(peers) if v := strings.ToLower(strings.TrimSpace(os.Getenv("WORKER_CLUSTER_BOOTSTRAP"))); v == "true" || v == "1" || v == "yes" { bootstrap = true } if bootstrap { opts.Bootstrap = true } else { opts.Seed = peers[0] } c, err := workercluster.New(opts) if err != nil { return nil, nil, fmt.Errorf("worker: cluster.New: %w", err) } if err := c.Start(ctx); err != nil { return nil, nil, fmt.Errorf("worker: cluster.Start: %w", err) } log.Printf("worker cluster: started node_id=%s addr=%s bootstrap=%t peers=%d", nodeID, localAddr, bootstrap, len(peers)) return c, &clusterAdapter{c: c}, nil } // clusterAdapter wraps *workercluster.Cluster so it implements the // webapp.ClusterView interface without webapp importing the raft // code path. type clusterAdapter struct { c *workercluster.Cluster } func (a *clusterAdapter) Stats() webapp.ClusterStats { src := a.c.ClusterStats() return webapp.ClusterStats{ NodeID: src.NodeID, LocalAddr: src.LocalAddr, State: src.State, Leader: src.Leader, Term: src.Term, AppliedIndex: src.AppliedIndex, LastIndex: src.LastIndex, NumPeers: src.NumPeers, Voters: src.Voters, FSMChecks: src.FSMChecks, FSMMembers: src.FSMMembers, FSMConfigVersion: src.FSMConfigVersion, FSMOutboxLen: src.FSMOutboxLen, FSMPartition: src.FSMPartition, } } func (a *clusterAdapter) ClusterID() string { return a.c.ClusterID() } func (a *clusterAdapter) LocalAddr() string { return a.c.LocalAddr() } // clusterModeEnabled returns true when WORKER_CLUSTER_ENABLED is set // to a truthy value. Kept as a free function (not a method) so the // webapp's ClusterEnabledFromEnv and the cmd binary agree on the // parsing rules. func clusterModeEnabled() bool { v := strings.ToLower(strings.TrimSpace(os.Getenv("WORKER_CLUSTER_ENABLED"))) return v == "true" || v == "1" || v == "yes" } // clusterPort derives the rafthttp bind port from WORKER_CLUSTER_PORT // or, when that env var is unset, WORKER_PORT+10000. The +10000 offset // keeps the webapp and the raft transport from colliding on the same // loopback bind. func clusterPort() string { if raw := strings.TrimSpace(os.Getenv("WORKER_CLUSTER_PORT")); raw != "" { return raw } base := distworker.DefaultHTTPPort if raw := strings.TrimSpace(os.Getenv("WORKER_PORT")); raw != "" { if v, err := strconv.Atoi(raw); err == nil && v > 0 && v <= 65535 { base = v } } return strconv.Itoa(base + 10000) } // parseClusterPeers parses a comma-separated list of "nodeID@host:port" // entries. Empty input returns an empty slice. Whitespace around entries // is trimmed; blank entries are rejected. func parseClusterPeers(raw string) ([]workercluster.Peer, error) { raw = strings.TrimSpace(raw) if raw == "" { return nil, nil } var out []workercluster.Peer for _, entry := range strings.Split(raw, ",") { entry = strings.TrimSpace(entry) if entry == "" { return nil, fmt.Errorf("empty peer entry in %q", raw) } at := strings.LastIndex(entry, "@") if at < 0 { return nil, fmt.Errorf("peer entry %q missing '@' separator (expected nodeID@host:port)", entry) } nodeID := strings.TrimSpace(entry[:at]) addr := strings.TrimSpace(entry[at+1:]) // Tolerate a scheme prefix; rafthttp is plain http. if i := strings.Index(addr, "://"); i >= 0 { addr = addr[i+3:] } if nodeID == "" || addr == "" { return nil, fmt.Errorf("peer entry %q has empty nodeID or address", entry) } if _, _, err := net.SplitHostPort(addr); err != nil { return nil, fmt.Errorf("peer entry %q: bad host:port: %w", entry, err) } out = append(out, workercluster.Peer{WorkerID: nodeID, Address: addr}) } return out, nil } // hasSeedPeer reports whether the peers list contains at least one // usable entry. Bootstrap nodes (the first node of a new cluster) have // an empty peers list. func hasSeedPeer(peers []workercluster.Peer) bool { return len(peers) > 0 } // runnerWrapper adapts *distworker.Runner to webapp.WorkerView. Kept // here (not in the webapp package) so the distworker -> webapp edge // is owned by the binary that links both packages. type runnerWrapper struct { r *distworker.Runner } func (w runnerWrapper) HTTPConfig() (cfg distworker.HTTPConfig) { if w.r == nil { return cfg } return w.r.HTTPConfig() } func (w runnerWrapper) Token() string { if w.r == nil { return "" } return w.r.Token() } func (w runnerWrapper) TokenRotatedAt() time.Time { if w.r == nil { return time.Time{} } return w.r.TokenRotatedAt() } func (w runnerWrapper) WorkerID() string { if w.r == nil { return "" } return w.r.WorkerID() } func (w runnerWrapper) RegionCode() string { if w.r == nil { return "" } return w.r.RegionCode() } func (w runnerWrapper) WorkerVersion() string { if w.r == nil { return "" } return w.r.WorkerVersion() } func (w runnerWrapper) WorkerCapabilities() []string { if w.r == nil { return nil } return w.r.WorkerCapabilities() } func (w runnerWrapper) LastHeartbeatAck() time.Time { if w.r == nil { return time.Time{} } return w.r.LastHeartbeatAck() } func (w runnerWrapper) MasterStatus() (*bool, time.Time) { if w.r == nil { return nil, time.Time{} } return w.r.MasterStatus() } func (w runnerWrapper) RecentResults(n int) []webapp.ResultRow { src := w.r.RecentResults(n) out := make([]webapp.ResultRow, len(src)) for i, r := range src { out[i] = webapp.ResultRow{ MonitorID: r.MonitorID, CheckID: r.CheckID, Kind: r.Kind, Host: r.Host, State: r.State, DurationMs: r.DurationMs, Error: r.Error, At: r.At, } } return out } func (w runnerWrapper) RecentNotifications(n int) []webapp.NotificationRow { src := w.r.RecentNotifications(n) out := make([]webapp.NotificationRow, len(src)) for i, r := range src { out[i] = webapp.NotificationRow{ Kind: r.Kind, Channel: r.Channel, Subject: r.Subject, Body: r.Body, OK: r.OK, Error: r.Error, At: r.At, JobID: r.JobID, Method: r.Method, Status: r.Status, DurationMs: r.DurationMs, } } return out } // logHTTPSettings prints a single line summarizing the HTTP listener // settings the operator configured, so misconfigurations are visible at // startup. Login is masked. willListen flips to true once the HTTP // listener is actually bound (Task 3) so the same helper can be reused. func logHTTPSettings(h distworker.HTTPConfig, willListen bool) { login := "***" if h.Login == "" { login = "(empty)" } url := h.PublicURL if url == "" { url = "(empty)" } log.Printf("worker http settings: host=%s port=%d public_url=%s login=%s will_listen=%t", h.Host, h.Port, url, login, willListen) if h.PublicURL != "" { if host, warn := distworker.WarnInsecurePublicURL(h.PublicURL); warn { label := distworker.EnvPublicURL if h.URLSource == distworker.PublicURLSourceLegacy { label = distworker.EnvWorkerURLLegacy } log.Printf( "worker http settings: WARN %s=http://%s uses plain HTTP on a non-loopback host; "+ "production deployments usually terminate TLS at a reverse proxy", label, host, ) } } if _, source := distworker.PublicURLFromEnv(); source == distworker.PublicURLSourceLegacy { log.Printf( "worker http settings: WARNING WORKER_URL is deprecated (bounded migration); " + "rename it to PUBLIC_URL before it is removed (docs/public-endpoint-and-identity.md milestone 1)", ) } } func loadDotEnv() { if err := godotenv.Load(".env"); err == nil { log.Println("worker .env file loaded") } } func healthCheck() int { endpoint := os.Getenv("RSMON_URL") if endpoint == "" { endpoint = "https://rsmon.ru" } endpoint = strings.TrimRight(endpoint, "/") if strings.HasSuffix(endpoint, "/api/worker") { endpoint = strings.TrimSuffix(endpoint, "/api/worker") } else if strings.HasSuffix(endpoint, "/worker") { endpoint = strings.TrimSuffix(endpoint, "/worker") } client := &http.Client{Timeout: 10 * time.Second} resp, err := client.Get(endpoint + "/up") if err != nil { fmt.Fprintf(os.Stderr, "health check failed: %v\n", err) return 1 } defer resp.Body.Close() //nolint:errcheck if resp.StatusCode != http.StatusOK { fmt.Fprintf(os.Stderr, "health check failed: status %s\n", resp.Status) return 1 } fmt.Println("health check ok") return 0 } func livenessCheck() int { httpCfg := distworker.HTTPConfigFromEnv() host := httpCfg.Host switch host { case "", "0.0.0.0": host = "127.0.0.1" case "::", "[::]": host = "::1" } endpoint := "http://" + net.JoinHostPort(host, strconv.Itoa(httpCfg.Port)) + "/healthz" return probeLiveness(endpoint) } func probeLiveness(endpoint string) int { client := &http.Client{Timeout: 5 * time.Second} resp, err := client.Get(endpoint) if err != nil { fmt.Fprintf(os.Stderr, "liveness check failed: %v\n", err) return 1 } defer resp.Body.Close() //nolint:errcheck if resp.StatusCode != http.StatusOK { fmt.Fprintf(os.Stderr, "liveness check failed: status %s\n", resp.Status) return 1 } fmt.Println("liveness check ok") return 0 }