worker/internal/distworker/server_metrics.go

package distworker

import (
	"context"
	"os"
	"path/filepath"
	"sort"
	"strconv"
	"strings"
	"sync"
	"syscall"
	"time"

	"rocketgit.ru/rsmon/worker/internal/wire"
)

type serverMetricSample struct {
	at                time.Time
	cpuTotal, cpuIdle uint64
	rx, tx            int64
}

var serverMetricSamples = struct {
	sync.Mutex
	byServer map[int64]serverMetricSample
}{byServer: make(map[int64]serverMetricSample)}

const serverMetricInterval = 5 * time.Second

const (
	maxServerMetricProcesses = 20
	maxServerMetricNetworks  = 16
)

// serverMetricLoop collects only on a worker assigned to a Server. It uses
// the real Linux /proc and statfs collector; unsupported platforms return no
// report rather than fabricated values. The worker has no control-plane DB
// access and forwards snapshots on its authenticated websocket.
func (r *Runner) serverMetricLoop(ctx context.Context) {
	ticker := time.NewTicker(serverMetricInterval)
	defer ticker.Stop()
	for {
		select {
		case <-ctx.Done():
			return
		case <-ticker.C:
			serverID := r.serverID.Load()
			if serverID == 0 {
				continue
			}
			report, ok := collectServerMetric(serverID)
			if !ok {
				continue
			}
			r.enqueueMetric(report)
		}
	}
}

func (r *Runner) enqueueMetric(report wire.ServerMetricReport) {
	if r.metricResults == nil {
		return
	}
	generation := r.metricGeneration.Load()
	if generation == 0 || r.metricGeneration.Load() != generation {
		return
	}
	select {
	case r.metricResults <- metricEnvelope{generation: generation, report: report}:
	default:
	}
}

func collectServerMetric(serverID int64) (wire.ServerMetricReport, bool) {
	memTotal, memAvailable, load1, load5, load15, uptime, ok := readLinuxHostMetrics("/proc")
	if !ok {
		return wire.ServerMetricReport{}, false
	}
	memUsed := memTotal - memAvailable
	var rxTotal, txTotal int64
	networks := make([]wire.NetworkMetric, 0)
	if data, err := os.ReadFile("/proc/net/dev"); err == nil {
		for _, line := range strings.Split(string(data), "\n") {
			fields := strings.Fields(line)
			if len(fields) >= 10 {
				name := strings.TrimSuffix(fields[0], ":")
				if name == "lo" {
					continue
				}
				rx, tx := parseI64(fields[1]), parseI64(fields[9])
				rxTotal += rx
				txTotal += tx
				networks = append(networks, wire.NetworkMetric{Interface: name, RxBytes: rx, TxBytes: tx})
			}
		}
	}
	sort.Slice(networks, func(i, j int) bool {
		return networks[i].RxBytes+networks[i].TxBytes > networks[j].RxBytes+networks[j].TxBytes
	})
	if len(networks) > maxServerMetricNetworks {
		networks = networks[:maxServerMetricNetworks]
	}
	processCount := countProcesses("/proc")
	processes := collectProcesses("/proc")
	now := time.Now()
	cpuPercent, netRx, netTx := sampledRates(serverID, now, rxTotal, txTotal)
	diskUsed, diskTotal, diskOK := rootDiskUsage()
	var diskUsedPtr, diskTotalPtr *int64
	if diskOK {
		diskUsedPtr, diskTotalPtr = &diskUsed, &diskTotal
	}
	return wire.ServerMetricReport{
		ServerID: serverID, CPUPercent: cpuPercent, MemUsed: &memUsed, MemTotal: &memTotal,
		DiskUsed: diskUsedPtr, DiskTotal: diskTotalPtr, NetRx: netRx, NetTx: netTx, HostUptimeSec: &uptime,
		Load1: &load1, Load5: &load5, Load15: &load15, ProcessCount: &processCount,
		Processes: processes, Networks: networks,
	}, true
}

// sampledRates turns monotonic /proc counters into percent and bytes/second.
// A first sample, a clock anomaly, or a counter reset intentionally has no rate.
func sampledRates(serverID int64, now time.Time, rx, tx int64) (*float64, *int64, *int64) {
	total, idle, ok := readCPUCounters("/proc/stat")
	if !ok {
		return nil, nil, nil
	}
	serverMetricSamples.Lock()
	defer serverMetricSamples.Unlock()
	previous, hasPrevious := serverMetricSamples.byServer[serverID]
	serverMetricSamples.byServer[serverID] = serverMetricSample{at: now, cpuTotal: total, cpuIdle: idle, rx: rx, tx: tx}
	if !hasPrevious || !now.After(previous.at) || total <= previous.cpuTotal || idle < previous.cpuIdle || rx < previous.rx || tx < previous.tx {
		return nil, nil, nil
	}
	cpu := float64((total-previous.cpuTotal)-(idle-previous.cpuIdle)) * 100 / float64(total-previous.cpuTotal)
	seconds := now.Sub(previous.at).Seconds()
	if seconds <= 0 {
		return &cpu, nil, nil
	}
	rxRate, txRate := int64(float64(rx-previous.rx)/seconds), int64(float64(tx-previous.tx)/seconds)
	return &cpu, &rxRate, &txRate
}

func readCPUCounters(path string) (uint64, uint64, bool) {
	data, err := os.ReadFile(path)
	if err != nil {
		return 0, 0, false
	}
	for _, line := range strings.Split(string(data), "\n") {
		if !strings.HasPrefix(line, "cpu ") {
			continue
		}
		fields := strings.Fields(line)
		if len(fields) < 5 {
			return 0, 0, false
		}
		var total uint64
		for _, field := range fields[1:] {
			value, err := strconv.ParseUint(field, 10, 64)
			if err != nil {
				return 0, 0, false
			}
			total += value
		}
		idle, _ := strconv.ParseUint(fields[4], 10, 64)
		if len(fields) > 5 {
			iowait, _ := strconv.ParseUint(fields[5], 10, 64)
			idle += iowait
		}
		return total, idle, total > 0
	}
	return 0, 0, false
}

func rootDiskUsage() (int64, int64, bool) {
	var stat syscall.Statfs_t
	if err := syscall.Statfs("/", &stat); err != nil || stat.Blocks == 0 {
		return 0, 0, false
	}
	total := int64(stat.Blocks * uint64(stat.Bsize))
	free := int64(stat.Bavail * uint64(stat.Bsize))
	return total - free, total, true
}

func collectProcesses(procRoot string) []wire.ProcessMetric {
	entries, err := os.ReadDir(procRoot)
	if err != nil {
		return nil
	}
	processes := make([]wire.ProcessMetric, 0, maxServerMetricProcesses)
	for _, entry := range entries {
		pid, err := strconv.Atoi(entry.Name())
		if err != nil {
			continue
		}
		status, err := os.ReadFile(filepath.Join(procRoot, entry.Name(), "status"))
		if err != nil {
			continue
		}
		name, rss := "", int64(0)
		for _, line := range strings.Split(string(status), "\n") {
			fields := strings.Fields(line)
			if len(fields) < 2 {
				continue
			}
			switch fields[0] {
			case "Name:":
				name = fields[1]
			case "VmRSS:":
				rss = parseI64(fields[1]) * 1024
			}
		}
		if name != "" {
			processes = append(processes, wire.ProcessMetric{PID: pid, Name: name, MemoryRSS: rss})
		}
	}
	sort.Slice(processes, func(i, j int) bool { return processes[i].MemoryRSS > processes[j].MemoryRSS })
	if len(processes) > maxServerMetricProcesses {
		processes = processes[:maxServerMetricProcesses]
	}
	return processes
}

func countProcesses(procRoot string) int {
	entries, err := os.ReadDir(procRoot)
	if err != nil {
		return 0
	}
	count := 0
	for _, entry := range entries {
		if _, err := strconv.Atoi(entry.Name()); err == nil {
			count++
		}
	}
	return count
}

func readLinuxHostMetrics(root string) (total, available int64, load1, load5, load15 float64, uptime int64, ok bool) {
	mem, err := os.ReadFile(filepath.Join(root, "meminfo"))
	if err != nil {
		return
	}
	for _, line := range strings.Split(string(mem), "\n") {
		fields := strings.Fields(line)
		if len(fields) < 2 {
			continue
		}
		switch fields[0] {
		case "MemTotal:":
			total = parseI64(fields[1]) * 1024
		case "MemAvailable:":
			available = parseI64(fields[1]) * 1024
		}
	}
	load, err := os.ReadFile(filepath.Join(root, "loadavg"))
	if err != nil {
		return
	}
	fields := strings.Fields(string(load))
	if len(fields) < 3 {
		return
	}
	load1, _ = strconv.ParseFloat(fields[0], 64)
	load5, _ = strconv.ParseFloat(fields[1], 64)
	load15, _ = strconv.ParseFloat(fields[2], 64)
	up, err := os.ReadFile(filepath.Join(root, "uptime"))
	if err != nil {
		return
	}
	fields = strings.Fields(string(up))
	if len(fields) == 0 {
		return
	}
	seconds, err := strconv.ParseFloat(fields[0], 64)
	if err != nil {
		return
	}
	uptime = int64(seconds)
	return total, available, load1, load5, load15, uptime, total > 0
}

func parseI64(value string) int64 { out, _ := strconv.ParseInt(value, 10, 64); return out }