feat: publish standalone worker
Separate worker packaging and service lifecycle from the control plane.
Этот коммит содержится в:
Gleb Tv
271
internal/distworker/server_metrics.go
Обычный файл
271
internal/distworker/server_metrics.go
Обычный файл
@@ -0,0 +1,271 @@
|
||||
package distworker
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"rsgit.ru/rsmon/rsmon/internal/wire"
|
||||
)
|
||||
|
||||
type serverMetricSample struct {
|
||||
at time.Time
|
||||
cpuTotal, cpuIdle uint64
|
||||
rx, tx int64
|
||||
}
|
||||
|
||||
var serverMetricSamples = struct {
|
||||
sync.Mutex
|
||||
byServer map[int64]serverMetricSample
|
||||
}{byServer: make(map[int64]serverMetricSample)}
|
||||
|
||||
const serverMetricInterval = 5 * time.Second
|
||||
|
||||
const (
|
||||
maxServerMetricProcesses = 20
|
||||
maxServerMetricNetworks = 16
|
||||
)
|
||||
|
||||
// serverMetricLoop collects only on a worker assigned to a Server. It uses
|
||||
// the real Linux /proc and statfs collector; unsupported platforms return no
|
||||
// report rather than fabricated values. The worker has no control-plane DB
|
||||
// access and forwards snapshots on its authenticated websocket.
|
||||
func (r *Runner) serverMetricLoop() {
|
||||
ticker := time.NewTicker(serverMetricInterval)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-r.stopCh:
|
||||
return
|
||||
case <-ticker.C:
|
||||
serverID := r.serverID.Load()
|
||||
if serverID == 0 || r.metricResults == nil {
|
||||
continue
|
||||
}
|
||||
report, ok := collectServerMetric(serverID)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
select {
|
||||
case r.metricResults <- report:
|
||||
case <-r.stopCh:
|
||||
return
|
||||
default:
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func collectServerMetric(serverID int64) (wire.ServerMetricReport, bool) {
|
||||
memTotal, memAvailable, load1, load5, load15, uptime, ok := readLinuxHostMetrics("/proc")
|
||||
if !ok {
|
||||
return wire.ServerMetricReport{}, false
|
||||
}
|
||||
memUsed := memTotal - memAvailable
|
||||
var rxTotal, txTotal int64
|
||||
networks := make([]wire.NetworkMetric, 0)
|
||||
if data, err := os.ReadFile("/proc/net/dev"); err == nil {
|
||||
for _, line := range strings.Split(string(data), "\n") {
|
||||
fields := strings.Fields(line)
|
||||
if len(fields) >= 10 {
|
||||
name := strings.TrimSuffix(fields[0], ":")
|
||||
if name == "lo" {
|
||||
continue
|
||||
}
|
||||
rx, tx := parseI64(fields[1]), parseI64(fields[9])
|
||||
rxTotal += rx
|
||||
txTotal += tx
|
||||
networks = append(networks, wire.NetworkMetric{Interface: name, RxBytes: rx, TxBytes: tx})
|
||||
}
|
||||
}
|
||||
}
|
||||
sort.Slice(networks, func(i, j int) bool {
|
||||
return networks[i].RxBytes+networks[i].TxBytes > networks[j].RxBytes+networks[j].TxBytes
|
||||
})
|
||||
if len(networks) > maxServerMetricNetworks {
|
||||
networks = networks[:maxServerMetricNetworks]
|
||||
}
|
||||
processCount := countProcesses("/proc")
|
||||
processes := collectProcesses("/proc")
|
||||
now := time.Now()
|
||||
cpuPercent, netRx, netTx := sampledRates(serverID, now, rxTotal, txTotal)
|
||||
diskUsed, diskTotal, diskOK := rootDiskUsage()
|
||||
var diskUsedPtr, diskTotalPtr *int64
|
||||
if diskOK {
|
||||
diskUsedPtr, diskTotalPtr = &diskUsed, &diskTotal
|
||||
}
|
||||
return wire.ServerMetricReport{
|
||||
ServerID: serverID, CPUPercent: cpuPercent, MemUsed: &memUsed, MemTotal: &memTotal,
|
||||
DiskUsed: diskUsedPtr, DiskTotal: diskTotalPtr, NetRx: netRx, NetTx: netTx, HostUptimeSec: &uptime,
|
||||
Load1: &load1, Load5: &load5, Load15: &load15, ProcessCount: &processCount,
|
||||
Processes: processes, Networks: networks,
|
||||
}, true
|
||||
}
|
||||
|
||||
// sampledRates turns monotonic /proc counters into percent and bytes/second.
|
||||
// A first sample, a clock anomaly, or a counter reset intentionally has no rate.
|
||||
func sampledRates(serverID int64, now time.Time, rx, tx int64) (*float64, *int64, *int64) {
|
||||
total, idle, ok := readCPUCounters("/proc/stat")
|
||||
if !ok {
|
||||
return nil, nil, nil
|
||||
}
|
||||
serverMetricSamples.Lock()
|
||||
defer serverMetricSamples.Unlock()
|
||||
previous, hasPrevious := serverMetricSamples.byServer[serverID]
|
||||
serverMetricSamples.byServer[serverID] = serverMetricSample{at: now, cpuTotal: total, cpuIdle: idle, rx: rx, tx: tx}
|
||||
if !hasPrevious || !now.After(previous.at) || total <= previous.cpuTotal || idle < previous.cpuIdle || rx < previous.rx || tx < previous.tx {
|
||||
return nil, nil, nil
|
||||
}
|
||||
cpu := float64((total-previous.cpuTotal)-(idle-previous.cpuIdle)) * 100 / float64(total-previous.cpuTotal)
|
||||
seconds := now.Sub(previous.at).Seconds()
|
||||
if seconds <= 0 {
|
||||
return &cpu, nil, nil
|
||||
}
|
||||
rxRate, txRate := int64(float64(rx-previous.rx)/seconds), int64(float64(tx-previous.tx)/seconds)
|
||||
return &cpu, &rxRate, &txRate
|
||||
}
|
||||
|
||||
func readCPUCounters(path string) (uint64, uint64, bool) {
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return 0, 0, false
|
||||
}
|
||||
for _, line := range strings.Split(string(data), "\n") {
|
||||
if !strings.HasPrefix(line, "cpu ") {
|
||||
continue
|
||||
}
|
||||
fields := strings.Fields(line)
|
||||
if len(fields) < 5 {
|
||||
return 0, 0, false
|
||||
}
|
||||
var total uint64
|
||||
for _, field := range fields[1:] {
|
||||
value, err := strconv.ParseUint(field, 10, 64)
|
||||
if err != nil {
|
||||
return 0, 0, false
|
||||
}
|
||||
total += value
|
||||
}
|
||||
idle, _ := strconv.ParseUint(fields[4], 10, 64)
|
||||
if len(fields) > 5 {
|
||||
iowait, _ := strconv.ParseUint(fields[5], 10, 64)
|
||||
idle += iowait
|
||||
}
|
||||
return total, idle, total > 0
|
||||
}
|
||||
return 0, 0, false
|
||||
}
|
||||
|
||||
func rootDiskUsage() (int64, int64, bool) {
|
||||
var stat syscall.Statfs_t
|
||||
if err := syscall.Statfs("/", &stat); err != nil || stat.Blocks == 0 {
|
||||
return 0, 0, false
|
||||
}
|
||||
total := int64(stat.Blocks * uint64(stat.Bsize))
|
||||
free := int64(stat.Bavail * uint64(stat.Bsize))
|
||||
return total - free, total, true
|
||||
}
|
||||
|
||||
func collectProcesses(procRoot string) []wire.ProcessMetric {
|
||||
entries, err := os.ReadDir(procRoot)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
processes := make([]wire.ProcessMetric, 0, maxServerMetricProcesses)
|
||||
for _, entry := range entries {
|
||||
pid, err := strconv.Atoi(entry.Name())
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
status, err := os.ReadFile(filepath.Join(procRoot, entry.Name(), "status"))
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
name, rss := "", int64(0)
|
||||
for _, line := range strings.Split(string(status), "\n") {
|
||||
fields := strings.Fields(line)
|
||||
if len(fields) < 2 {
|
||||
continue
|
||||
}
|
||||
switch fields[0] {
|
||||
case "Name:":
|
||||
name = fields[1]
|
||||
case "VmRSS:":
|
||||
rss = parseI64(fields[1]) * 1024
|
||||
}
|
||||
}
|
||||
if name != "" {
|
||||
processes = append(processes, wire.ProcessMetric{PID: pid, Name: name, MemoryRSS: rss})
|
||||
}
|
||||
}
|
||||
sort.Slice(processes, func(i, j int) bool { return processes[i].MemoryRSS > processes[j].MemoryRSS })
|
||||
if len(processes) > maxServerMetricProcesses {
|
||||
processes = processes[:maxServerMetricProcesses]
|
||||
}
|
||||
return processes
|
||||
}
|
||||
|
||||
func countProcesses(procRoot string) int {
|
||||
entries, err := os.ReadDir(procRoot)
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
count := 0
|
||||
for _, entry := range entries {
|
||||
if _, err := strconv.Atoi(entry.Name()); err == nil {
|
||||
count++
|
||||
}
|
||||
}
|
||||
return count
|
||||
}
|
||||
|
||||
func readLinuxHostMetrics(root string) (total, available int64, load1, load5, load15 float64, uptime int64, ok bool) {
|
||||
mem, err := os.ReadFile(filepath.Join(root, "meminfo"))
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
for _, line := range strings.Split(string(mem), "\n") {
|
||||
fields := strings.Fields(line)
|
||||
if len(fields) < 2 {
|
||||
continue
|
||||
}
|
||||
switch fields[0] {
|
||||
case "MemTotal:":
|
||||
total = parseI64(fields[1]) * 1024
|
||||
case "MemAvailable:":
|
||||
available = parseI64(fields[1]) * 1024
|
||||
}
|
||||
}
|
||||
load, err := os.ReadFile(filepath.Join(root, "loadavg"))
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
fields := strings.Fields(string(load))
|
||||
if len(fields) < 3 {
|
||||
return
|
||||
}
|
||||
load1, _ = strconv.ParseFloat(fields[0], 64)
|
||||
load5, _ = strconv.ParseFloat(fields[1], 64)
|
||||
load15, _ = strconv.ParseFloat(fields[2], 64)
|
||||
up, err := os.ReadFile(filepath.Join(root, "uptime"))
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
fields = strings.Fields(string(up))
|
||||
if len(fields) == 0 {
|
||||
return
|
||||
}
|
||||
seconds, err := strconv.ParseFloat(fields[0], 64)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
uptime = int64(seconds)
|
||||
return total, available, load1, load5, load15, uptime, total > 0
|
||||
}
|
||||
|
||||
func parseI64(value string) int64 { out, _ := strconv.ParseInt(value, 10, 64); return out }
|
||||
Ссылка в новой задаче
Block a user