package distworker import ( "context" "os" "path/filepath" "sort" "strconv" "strings" "sync" "syscall" "time" "rocketgit.ru/rsmon/worker/internal/wire" ) type serverMetricSample struct { at time.Time cpuTotal, cpuIdle uint64 rx, tx int64 } var serverMetricSamples = struct { sync.Mutex byServer map[int64]serverMetricSample }{byServer: make(map[int64]serverMetricSample)} const serverMetricInterval = 5 * time.Second const ( maxServerMetricProcesses = 20 maxServerMetricNetworks = 16 ) // serverMetricLoop collects only on a worker assigned to a Server. It uses // the real Linux /proc and statfs collector; unsupported platforms return no // report rather than fabricated values. The worker has no control-plane DB // access and forwards snapshots on its authenticated websocket. func (r *Runner) serverMetricLoop(ctx context.Context) { ticker := time.NewTicker(serverMetricInterval) defer ticker.Stop() for { select { case <-ctx.Done(): return case <-ticker.C: serverID := r.serverID.Load() if serverID == 0 { continue } report, ok := collectServerMetric(serverID) if !ok { continue } r.enqueueMetric(report) } } } func (r *Runner) enqueueMetric(report wire.ServerMetricReport) { if r.metricResults == nil { return } generation := r.metricGeneration.Load() if generation == 0 || r.metricGeneration.Load() != generation { return } select { case r.metricResults <- metricEnvelope{generation: generation, report: report}: default: } } func collectServerMetric(serverID int64) (wire.ServerMetricReport, bool) { memTotal, memAvailable, load1, load5, load15, uptime, ok := readLinuxHostMetrics("/proc") if !ok { return wire.ServerMetricReport{}, false } memUsed := memTotal - memAvailable var rxTotal, txTotal int64 networks := make([]wire.NetworkMetric, 0) if data, err := os.ReadFile("/proc/net/dev"); err == nil { for _, line := range strings.Split(string(data), "\n") { fields := strings.Fields(line) if len(fields) >= 10 { name := strings.TrimSuffix(fields[0], ":") if name == "lo" { continue } rx, tx := parseI64(fields[1]), parseI64(fields[9]) rxTotal += rx txTotal += tx networks = append(networks, wire.NetworkMetric{Interface: name, RxBytes: rx, TxBytes: tx}) } } } sort.Slice(networks, func(i, j int) bool { return networks[i].RxBytes+networks[i].TxBytes > networks[j].RxBytes+networks[j].TxBytes }) if len(networks) > maxServerMetricNetworks { networks = networks[:maxServerMetricNetworks] } processCount := countProcesses("/proc") processes := collectProcesses("/proc") now := time.Now() cpuPercent, netRx, netTx := sampledRates(serverID, now, rxTotal, txTotal) diskUsed, diskTotal, diskOK := rootDiskUsage() var diskUsedPtr, diskTotalPtr *int64 if diskOK { diskUsedPtr, diskTotalPtr = &diskUsed, &diskTotal } return wire.ServerMetricReport{ ServerID: serverID, CPUPercent: cpuPercent, MemUsed: &memUsed, MemTotal: &memTotal, DiskUsed: diskUsedPtr, DiskTotal: diskTotalPtr, NetRx: netRx, NetTx: netTx, HostUptimeSec: &uptime, Load1: &load1, Load5: &load5, Load15: &load15, ProcessCount: &processCount, Processes: processes, Networks: networks, }, true } // sampledRates turns monotonic /proc counters into percent and bytes/second. // A first sample, a clock anomaly, or a counter reset intentionally has no rate. func sampledRates(serverID int64, now time.Time, rx, tx int64) (*float64, *int64, *int64) { total, idle, ok := readCPUCounters("/proc/stat") if !ok { return nil, nil, nil } serverMetricSamples.Lock() defer serverMetricSamples.Unlock() previous, hasPrevious := serverMetricSamples.byServer[serverID] serverMetricSamples.byServer[serverID] = serverMetricSample{at: now, cpuTotal: total, cpuIdle: idle, rx: rx, tx: tx} if !hasPrevious || !now.After(previous.at) || total <= previous.cpuTotal || idle < previous.cpuIdle || rx < previous.rx || tx < previous.tx { return nil, nil, nil } cpu := float64((total-previous.cpuTotal)-(idle-previous.cpuIdle)) * 100 / float64(total-previous.cpuTotal) seconds := now.Sub(previous.at).Seconds() if seconds <= 0 { return &cpu, nil, nil } rxRate, txRate := int64(float64(rx-previous.rx)/seconds), int64(float64(tx-previous.tx)/seconds) return &cpu, &rxRate, &txRate } func readCPUCounters(path string) (uint64, uint64, bool) { data, err := os.ReadFile(path) if err != nil { return 0, 0, false } for _, line := range strings.Split(string(data), "\n") { if !strings.HasPrefix(line, "cpu ") { continue } fields := strings.Fields(line) if len(fields) < 5 { return 0, 0, false } var total uint64 for _, field := range fields[1:] { value, err := strconv.ParseUint(field, 10, 64) if err != nil { return 0, 0, false } total += value } idle, _ := strconv.ParseUint(fields[4], 10, 64) if len(fields) > 5 { iowait, _ := strconv.ParseUint(fields[5], 10, 64) idle += iowait } return total, idle, total > 0 } return 0, 0, false } func rootDiskUsage() (int64, int64, bool) { var stat syscall.Statfs_t if err := syscall.Statfs("/", &stat); err != nil || stat.Blocks == 0 { return 0, 0, false } total := int64(stat.Blocks * uint64(stat.Bsize)) free := int64(stat.Bavail * uint64(stat.Bsize)) return total - free, total, true } func collectProcesses(procRoot string) []wire.ProcessMetric { entries, err := os.ReadDir(procRoot) if err != nil { return nil } processes := make([]wire.ProcessMetric, 0, maxServerMetricProcesses) for _, entry := range entries { pid, err := strconv.Atoi(entry.Name()) if err != nil { continue } status, err := os.ReadFile(filepath.Join(procRoot, entry.Name(), "status")) if err != nil { continue } name, rss := "", int64(0) for _, line := range strings.Split(string(status), "\n") { fields := strings.Fields(line) if len(fields) < 2 { continue } switch fields[0] { case "Name:": name = fields[1] case "VmRSS:": rss = parseI64(fields[1]) * 1024 } } if name != "" { processes = append(processes, wire.ProcessMetric{PID: pid, Name: name, MemoryRSS: rss}) } } sort.Slice(processes, func(i, j int) bool { return processes[i].MemoryRSS > processes[j].MemoryRSS }) if len(processes) > maxServerMetricProcesses { processes = processes[:maxServerMetricProcesses] } return processes } func countProcesses(procRoot string) int { entries, err := os.ReadDir(procRoot) if err != nil { return 0 } count := 0 for _, entry := range entries { if _, err := strconv.Atoi(entry.Name()); err == nil { count++ } } return count } func readLinuxHostMetrics(root string) (total, available int64, load1, load5, load15 float64, uptime int64, ok bool) { mem, err := os.ReadFile(filepath.Join(root, "meminfo")) if err != nil { return } for _, line := range strings.Split(string(mem), "\n") { fields := strings.Fields(line) if len(fields) < 2 { continue } switch fields[0] { case "MemTotal:": total = parseI64(fields[1]) * 1024 case "MemAvailable:": available = parseI64(fields[1]) * 1024 } } load, err := os.ReadFile(filepath.Join(root, "loadavg")) if err != nil { return } fields := strings.Fields(string(load)) if len(fields) < 3 { return } load1, _ = strconv.ParseFloat(fields[0], 64) load5, _ = strconv.ParseFloat(fields[1], 64) load15, _ = strconv.ParseFloat(fields[2], 64) up, err := os.ReadFile(filepath.Join(root, "uptime")) if err != nil { return } fields = strings.Fields(string(up)) if len(fields) == 0 { return } seconds, err := strconv.ParseFloat(fields[0], 64) if err != nil { return } uptime = int64(seconds) return total, available, load1, load5, load15, uptime, total > 0 } func parseI64(value string) int64 { out, _ := strconv.ParseInt(value, 10, 64); return out }