worker/cmd/rsmon-worker/main.go

// Distributed monitoring worker binary.
package main

import (
	"context"
	"flag"
	"fmt"
	"log"
	"net"
	"net/http"
	"os"
	"os/signal"
	"strconv"
	"strings"
	"syscall"
	"time"

	"github.com/hashicorp/raft"
	"github.com/joho/godotenv"

	"rocketgit.ru/rsmon/worker/internal/distworker"
	"rocketgit.ru/rsmon/worker/internal/webapp"
	"rocketgit.ru/rsmon/worker/internal/workercluster"
)

var (
	// Build info set by ldflags.
	version   = "dev"
	commit    = "unknown"
	buildDate = "unknown"

	// webappEnabled flips the local web UI on at boot. Default true.
	// Phase 1 keeps it on; the flag exists so a Phase 2 basic-auth
	// install can opt out without recompiling.
	webappEnabled = true

	// clusterDebugApplyTestConfig, when true, submits the hardcoded
	// CriticalCheckConfig from workercluster.DefaultDebugCriticalCheck
	// to the cluster on startup. Wired via the
	// --cluster-debug-apply-test-config CLI flag; the e2e script
	// uses this so it can verify FSM replication without the
	// signed-config-adoption producer (which lands in a later phase).
	//
	// DEBUG: this flag is a placeholder. It must be removed (or
	// guarded behind a build tag) before any production build.
	//
	// TODO(phase-N): remove once the real producer is wired.
	clusterDebugApplyTestConfig = false
)

func main() {
	log.SetFlags(log.LstdFlags | log.Lshortfile)
	loadDotEnv()

	versionFlag := flag.Bool("version", false, "Print version and exit")
	noWeb := flag.Bool("no-web", false, "Disable the local web UI (Phase 1 ships with it on)")
	debugApplyConfig := flag.Bool("cluster-debug-apply-test-config", false,
		"Submit a hardcoded CriticalCheckConfig to the cluster on startup. DEBUG: remove once the real config.adopt producer is wired.")
	flag.Parse()

	if *versionFlag {
		fmt.Printf("rsmon-worker version=%s commit=%s buildDate=%s\n", version, commit, buildDate)
		os.Exit(0)
	}
	webappEnabled = !*noWeb
	clusterDebugApplyTestConfig = *debugApplyConfig

	if len(flag.Args()) > 0 && flag.Arg(0) == "health" {
		os.Exit(healthCheck())
	}

	log.Println("rsmon-worker starting...")

	cfg := distworker.ConfigFromEnv()
	logHTTPSettings(cfg.HTTP, webappEnabled)
	// The HTTP listener is started by the webapp. WORKER_LOGIN /
	// WORKER_PASSWORD (basic auth) are passed through to the webapp
	// Config; the webapp's ValidateBasicAuth rejects XOR.
	if err := webapp.ValidateBasicAuth(cfg.HTTP.Login, cfg.HTTP.Password); err != nil {
		log.Fatalf("worker: %v", err)
	}

	runner := distworker.NewRunner(&cfg)

	// Graceful shutdown context shared by the runner, the cluster,
	// and the webapp. ctxCancel is called by the signal handler so
	// all three wind down together; the runner waits for its
	// goroutines, the cluster drains its rafthttp listener + raft
	// state machine, then the webapp closes its listener and the
	// SQLite handle. The defer is a safety net for early returns
	// before the signal handler registers (the handler always wins
	// for SIGINT/SIGTERM, but other early exits rely on the defer).
	ctx, ctxCancel := context.WithCancel(context.Background())
	defer func() { ctxCancel() }() //nolint:gocritic // safety net for early returns; signal handler owns the canonical path

	sigCh := make(chan os.Signal, 1)
	signal.Notify(sigCh, syscall.SIGINT, syscall.SIGTERM)
	go func() {
		<-sigCh
		log.Println("worker: shutdown signal received")
		runner.Stop()
		ctxCancel()
	}()

	// Construct the cluster subsystem first (Task 5) so its admin
	// endpoints can be wired into the webapp. The cluster is optional;
	// when WORKER_CLUSTER_ENABLED=false (the default) we skip it and
	// the webapp falls back to the local-only auth path.
	cluster, clusterView, err := buildCluster(ctx, &cfg)
	if err != nil {
		// buildCluster never returns a partial cluster on error, so
		// nothing to clean up here. Print and exit so the trailing
		// defer (which only runs when cluster != nil) does not
		// confuse linters or runtime observers.
		log.Printf("worker: cluster init failed: %v", err)
		os.Exit(1) //nolint:gocritic // safety net; see comment above
	}
	if cluster != nil {
		defer func() {
			shut, cancel := context.WithTimeout(context.Background(), 5*time.Second)
			defer cancel()
			if err := cluster.Shutdown(shut); err != nil {
				log.Printf("worker: cluster shutdown: %v", err)
			}
		}()
	}

	// Run the runner and the webapp concurrently. The runner blocks
	// until ctxCancel; the webapp is started in its own goroutine
	// so a web-side error does not block the runner. Both unwind
	// when ctxCancel fires.
	webappErrCh := make(chan error, 1)
	if webappEnabled {
		srv, _, err := buildWebapp(ctx, runner, clusterView)
		if err != nil {
			// No deferred cleanup needed: the runner has not been
			// started yet, the signal handler has not registered,
			// and the only shared resource is the context which
			// has nothing tied to it. log.Fatalf calls os.Exit so
			// the deferred ctxCancel would be redundant noise.
			log.Fatalf("worker: webapp init failed: %v", err)
		}
		go func() {
			webappErrCh <- srv.Start(ctx)
		}()
		// Provision the first-run user before the runner gets a
		// chance to send its first websocket hello so the operator
		// can log in immediately if the main app is slow to ack.
		if err := webapp.ProvisionFirstRunIfNeeded(srv, log.New(os.Stderr, "webapp: ", log.LstdFlags)); err != nil {
			log.Printf("worker: webapp first-run provisioning: %v", err)
		}
	}

	if err := runner.Start(); err != nil {
		log.Fatal("worker failed:", err)
	}

	// Runner returned: the signal handler already canceled ctx so
	// the webapp goroutine will exit shortly. Wait for it to avoid
	// leaking the SQLite handle.
	if webappEnabled {
		select {
		case err := <-webappErrCh:
			if err != nil {
				log.Printf("worker: webapp exited: %v", err)
			}
		case <-time.After(5 * time.Second):
			log.Printf("worker: webapp shutdown timed out")
		}
	}
}

// buildWebapp wires the worker view into a webapp.Server. The Deps
// adapter reads from the runner (which is not yet Started at the
// time this is called; recent buffers are empty by design).
func buildWebapp(_ context.Context, runner *distworker.Runner, cluster webapp.ClusterView) (*webapp.Server, *webapp.Deps, error) {
	deps := &webapp.Deps{
		Runner:    runnerWrapper{runner},
		Cluster:   cluster,
		Version:   version,
		BuildDate: buildDate,
		Commit:    commit,
		StartedAt: time.Now().UTC(),
		Logger:    log.New(os.Stderr, "webapp: ", log.LstdFlags|log.Lshortfile),
		TokenRotator: func(ctx context.Context) (string, error) {
			return runner.RotateToken(ctx)
		},
		ReleaseHTTPClient: &http.Client{
			Timeout: 5 * time.Second,
			Transport: &http.Transport{
				Proxy: http.ProxyFromEnvironment,
			},
		},
	}
	srv, err := webapp.New(webapp.ConfigFromEnvOrDefault(), deps)
	if err != nil {
		return nil, deps, err
	}
	return srv, deps, nil
}

// buildCluster reads WORKER_CLUSTER_* env vars and, when cluster mode
// is enabled, constructs and starts a *workercluster.Cluster. The
// returned ClusterView is the narrow interface webapp consumes; it is
// nil when the cluster is not enabled.
//
// The cluster's rafthttp listener binds to 127.0.0.1 on
// WORKER_CLUSTER_PORT (default = WORKER_PORT + 10000) so the
// worker webapp listener and the raft transport do not collide. The
// peers list (WORKER_CLUSTER_PEERS) is parsed as a comma-separated
// list of "nodeID@host:port" entries; the first peer becomes the
// Seed for non-bootstrap nodes.
//
// The function never returns a partial cluster: either both the
// concrete *Cluster and the ClusterView are returned, or both are nil
// (cluster disabled) or an error is returned (cluster enabled but
// misconfigured).
func buildCluster(ctx context.Context, cfg *distworker.Config) (*workercluster.Cluster, webapp.ClusterView, error) {
	if !clusterModeEnabled() {
		return nil, nil, nil
	}
	creds := workercluster.HTTPCreds{Login: cfg.HTTP.Login, Password: cfg.HTTP.Password}
	if !creds.IsConfigured() {
		return nil, nil, fmt.Errorf(
			"worker: WORKER_CLUSTER_ENABLED=true requires WORKER_LOGIN and WORKER_PASSWORD (rafthttp basic auth)")
	}

	nodeID := strings.TrimSpace(os.Getenv("WORKER_CLUSTER_ID"))
	if nodeID == "" {
		return nil, nil, fmt.Errorf("worker: WORKER_CLUSTER_ID is required when WORKER_CLUSTER_ENABLED=true")
	}
	dataDir := strings.TrimSpace(os.Getenv("WORKER_CLUSTER_DATA_DIR"))
	if dataDir == "" {
		return nil, nil, fmt.Errorf("worker: WORKER_CLUSTER_DATA_DIR is required when WORKER_CLUSTER_ENABLED=true")
	}
	if err := workercluster.EnsureDataDir(dataDir); err != nil {
		return nil, nil, fmt.Errorf("worker: WORKER_CLUSTER_DATA_DIR %q: %w", dataDir, err)
	}

	port := clusterPort()
	host := strings.TrimSpace(os.Getenv("WORKER_CLUSTER_HOST"))
	if host == "" {
		host = "127.0.0.1"
	}
	localAddr := net.JoinHostPort(host, port)

	peers, err := parseClusterPeers(os.Getenv("WORKER_CLUSTER_PEERS"))
	if err != nil {
		return nil, nil, fmt.Errorf("worker: WORKER_CLUSTER_PEERS: %w", err)
	}

	opts := &workercluster.Options{
		NodeID:           nodeID,
		LocalAddr:        localAddr,
		DataDir:          dataDir,
		Creds:            creds,
		HeartbeatTimeout: 1000 * time.Millisecond,
		ElectionTimeout:  3000 * time.Millisecond,
		Logger:           log.New(os.Stderr, "[workercluster] ", log.LstdFlags),
		LogOutput:        os.Stderr,
	}
	// WORKER_CLUSTER_BOOTSTRAP=true forces bootstrap mode even
	// when WORKER_CLUSTER_PEERS is set (the peers list is then
	// informational; the cluster subsystem records it but does
	// not dial). Without it, a non-empty peers list means the
	// node joins via the first peer. An empty peers list always
	// bootstraps.
	bootstrap := !hasSeedPeer(peers)
	if v := strings.ToLower(strings.TrimSpace(os.Getenv("WORKER_CLUSTER_BOOTSTRAP"))); v == "true" || v == "1" || v == "yes" {
		bootstrap = true
	}
	if bootstrap {
		opts.Bootstrap = true
	} else {
		opts.Seed = peers[0]
	}

	c, err := workercluster.New(opts)
	if err != nil {
		return nil, nil, fmt.Errorf("worker: cluster.New: %w", err)
	}
	if err := c.Start(ctx); err != nil {
		return nil, nil, fmt.Errorf("worker: cluster.Start: %w", err)
	}
	log.Printf("worker cluster: started node_id=%s addr=%s bootstrap=%t peers=%d",
		nodeID, localAddr, bootstrap, len(peers))

	if clusterDebugApplyTestConfig && bootstrap {
		// The debug apply only fires on bootstrap nodes; a
		// joiner cannot commit a log entry until it has been
		// promoted to voter. Wait for this node to win an
		// election first (a fresh single-voter cluster elects
		// itself immediately but the goroutine may run before
		// the state has flipped).
		go func() {
			deadline := time.Now().Add(15 * time.Second)
			for time.Now().Before(deadline) {
				if c.Raft() != nil && c.Raft().State() == raft.Leader {
					break
				}
				time.Sleep(100 * time.Millisecond)
			}
			if c.Raft() == nil || c.Raft().State() != raft.Leader {
				log.Printf("worker cluster: applying debug test config id=%d: not leader after 15s",
					workercluster.DefaultDebugCriticalCheck().ID)
				return
			}
			check := workercluster.DefaultDebugCriticalCheck()
			applied, err := c.ApplyTestConfig(&check)
			if err != nil {
				log.Printf("worker cluster: applying debug test config id=%d: %v",
					check.ID, err)
				return
			}
			log.Printf("worker cluster: applying debug test config id=%d applied_index=%d",
				check.ID, applied)
		}()
	}

	return c, &clusterAdapter{c: c}, nil
}

// clusterAdapter wraps *workercluster.Cluster so it implements the
// webapp.ClusterView interface without webapp importing the raft
// code path. The ApplyTestConfig signature is the one webapp expects
// (no config argument; the cluster subsystem owns the hardcoded
// payload so the two sides cannot drift).
type clusterAdapter struct {
	c *workercluster.Cluster
}

func (a *clusterAdapter) Stats() webapp.ClusterStats {
	src := a.c.ClusterStats()
	return webapp.ClusterStats{
		NodeID:           src.NodeID,
		LocalAddr:        src.LocalAddr,
		State:            src.State,
		Leader:           src.Leader,
		Term:             src.Term,
		AppliedIndex:     src.AppliedIndex,
		LastIndex:        src.LastIndex,
		NumPeers:         src.NumPeers,
		Voters:           src.Voters,
		FSMChecks:        src.FSMChecks,
		FSMMembers:       src.FSMMembers,
		FSMConfigVersion: src.FSMConfigVersion,
		FSMOutboxLen:     src.FSMOutboxLen,
		FSMPartition:     src.FSMPartition,
	}
}

func (a *clusterAdapter) ApplyTestConfig() (uint64, error) {
	check := workercluster.DefaultDebugCriticalCheck()
	return a.c.ApplyTestConfig(&check)
}

func (a *clusterAdapter) ClusterID() string { return a.c.ClusterID() }
func (a *clusterAdapter) LocalAddr() string { return a.c.LocalAddr() }

// clusterModeEnabled returns true when WORKER_CLUSTER_ENABLED is set
// to a truthy value. Kept as a free function (not a method) so the
// webapp's ClusterEnabledFromEnv and the cmd binary agree on the
// parsing rules.
func clusterModeEnabled() bool {
	v := strings.ToLower(strings.TrimSpace(os.Getenv("WORKER_CLUSTER_ENABLED")))
	return v == "true" || v == "1" || v == "yes"
}

// clusterPort derives the rafthttp bind port from WORKER_CLUSTER_PORT
// or, when that env var is unset, WORKER_PORT+10000. The +10000 offset
// keeps the webapp and the raft transport from colliding on the same
// loopback bind.
func clusterPort() string {
	if raw := strings.TrimSpace(os.Getenv("WORKER_CLUSTER_PORT")); raw != "" {
		return raw
	}
	base := distworker.DefaultHTTPPort
	if raw := strings.TrimSpace(os.Getenv("WORKER_PORT")); raw != "" {
		if v, err := strconv.Atoi(raw); err == nil && v > 0 && v <= 65535 {
			base = v
		}
	}
	return strconv.Itoa(base + 10000)
}

// parseClusterPeers parses a comma-separated list of "nodeID@host:port"
// entries. Empty input returns an empty slice. Whitespace around entries
// is trimmed; blank entries are rejected.
func parseClusterPeers(raw string) ([]workercluster.Peer, error) {
	raw = strings.TrimSpace(raw)
	if raw == "" {
		return nil, nil
	}
	var out []workercluster.Peer
	for _, entry := range strings.Split(raw, ",") {
		entry = strings.TrimSpace(entry)
		if entry == "" {
			return nil, fmt.Errorf("empty peer entry in %q", raw)
		}
		at := strings.LastIndex(entry, "@")
		if at < 0 {
			return nil, fmt.Errorf("peer entry %q missing '@' separator (expected nodeID@host:port)", entry)
		}
		nodeID := strings.TrimSpace(entry[:at])
		addr := strings.TrimSpace(entry[at+1:])
		// Tolerate a scheme prefix; rafthttp is plain http.
		if i := strings.Index(addr, "://"); i >= 0 {
			addr = addr[i+3:]
		}
		if nodeID == "" || addr == "" {
			return nil, fmt.Errorf("peer entry %q has empty nodeID or address", entry)
		}
		if _, _, err := net.SplitHostPort(addr); err != nil {
			return nil, fmt.Errorf("peer entry %q: bad host:port: %w", entry, err)
		}
		out = append(out, workercluster.Peer{WorkerID: nodeID, Address: addr})
	}
	return out, nil
}

// hasSeedPeer reports whether the peers list contains at least one
// usable entry. Bootstrap nodes (the first node of a new cluster) have
// an empty peers list.
func hasSeedPeer(peers []workercluster.Peer) bool {
	return len(peers) > 0
}

// runnerWrapper adapts *distworker.Runner to webapp.WorkerView. Kept
// here (not in the webapp package) so the distworker -> webapp edge
// is owned by the binary that links both packages.
type runnerWrapper struct {
	r *distworker.Runner
}

func (w runnerWrapper) HTTPConfig() (cfg distworker.HTTPConfig) {
	if w.r == nil {
		return cfg
	}
	return w.r.HTTPConfig()
}

func (w runnerWrapper) Token() string {
	if w.r == nil {
		return ""
	}
	return w.r.Token()
}

func (w runnerWrapper) TokenRotatedAt() time.Time {
	if w.r == nil {
		return time.Time{}
	}
	return w.r.TokenRotatedAt()
}

func (w runnerWrapper) WorkerID() string {
	if w.r == nil {
		return ""
	}
	return w.r.WorkerID()
}

func (w runnerWrapper) RegionCode() string {
	if w.r == nil {
		return ""
	}
	return w.r.RegionCode()
}

func (w runnerWrapper) WorkerVersion() string {
	if w.r == nil {
		return ""
	}
	return w.r.WorkerVersion()
}

func (w runnerWrapper) WorkerCapabilities() []string {
	if w.r == nil {
		return nil
	}
	return w.r.WorkerCapabilities()
}

func (w runnerWrapper) LastHeartbeatAck() time.Time {
	if w.r == nil {
		return time.Time{}
	}
	return w.r.LastHeartbeatAck()
}

func (w runnerWrapper) MasterStatus() (*bool, time.Time) {
	if w.r == nil {
		return nil, time.Time{}
	}
	return w.r.MasterStatus()
}

func (w runnerWrapper) RecentResults(n int) []webapp.ResultRow {
	src := w.r.RecentResults(n)
	out := make([]webapp.ResultRow, len(src))
	for i, r := range src {
		out[i] = webapp.ResultRow{
			MonitorID:  r.MonitorID,
			CheckID:    r.CheckID,
			Kind:       r.Kind,
			Host:       r.Host,
			State:      r.State,
			DurationMs: r.DurationMs,
			Error:      r.Error,
			At:         r.At,
		}
	}
	return out
}

func (w runnerWrapper) RecentNotifications(n int) []webapp.NotificationRow {
	src := w.r.RecentNotifications(n)
	out := make([]webapp.NotificationRow, len(src))
	for i, r := range src {
		out[i] = webapp.NotificationRow{
			Kind:    r.Kind,
			Channel: r.Channel,
			Subject: r.Subject,
			Body:    r.Body,
			OK:      r.OK,
			Error:   r.Error,
			At:      r.At,
		}
	}
	return out
}

// logHTTPSettings prints a single line summarizing the HTTP listener
// settings the operator configured, so misconfigurations are visible at
// startup. Login is masked. willListen flips to true once the HTTP
// listener is actually bound (Task 3) so the same helper can be reused.
func logHTTPSettings(h distworker.HTTPConfig, willListen bool) {
	login := "***"
	if h.Login == "" {
		login = "(empty)"
	}
	url := h.URL
	if url == "" {
		url = "(empty)"
	}
	log.Printf("worker http settings: host=%s port=%d url=%s login=%s will_listen=%t",
		h.Host, h.Port, url, login, willListen)
	if h.URL != "" {
		if host, warn := distworker.WarnInsecurePublicURL(h.URL); warn {
			log.Printf(
				"worker http settings: WARN WORKER_URL=http://%s uses plain HTTP on a non-loopback host; "+
					"production deployments usually terminate TLS at a reverse proxy", host,
			)
		}
	}
}

func loadDotEnv() {
	if err := godotenv.Load(".env"); err == nil {
		log.Println("worker .env file loaded")
	}
}

func healthCheck() int {
	endpoint := os.Getenv("RSMON_URL")
	if endpoint == "" {
		endpoint = "https://rsmon.ru"
	}
	endpoint = strings.TrimRight(endpoint, "/")
	if strings.HasSuffix(endpoint, "/api/worker") {
		endpoint = strings.TrimSuffix(endpoint, "/api/worker")
	} else if strings.HasSuffix(endpoint, "/worker") {
		endpoint = strings.TrimSuffix(endpoint, "/worker")
	}
	client := &http.Client{Timeout: 10 * time.Second}
	resp, err := client.Get(endpoint + "/up")
	if err != nil {
		fmt.Fprintf(os.Stderr, "health check failed: %v\n", err)
		return 1
	}
	defer resp.Body.Close() //nolint:errcheck
	if resp.StatusCode != http.StatusOK {
		fmt.Fprintf(os.Stderr, "health check failed: status %s\n", resp.Status)
		return 1
	}
	fmt.Println("health check ok")
	return 0
}