feat: publish standalone worker

Separate worker packaging and service lifecycle from the control plane.

checks/cdns/a_records.go

@@ -0,0 +1,39 @@
+// Package cdns provides functionality.
+package cdns
+
+import (
+	"errors"
+	"strconv"
+
+	"github.com/miekg/dns"
+
+	"rsgit.ru/rsmon/rsmon/internal/netaddr"
+)
+
+// FetchARecords provides functionality.
+func FetchARecords(zone, ns string) ([]NSRecord, error) {
+	config := dns.ClientConfig{Servers: []string{ns}}
+	c := new(dns.Client)
+	m := new(dns.Msg)
+	m.SetQuestion(zone, dns.TypeA)
+	m.RecursionDesired = true
+	r, _, err := c.Exchange(m, config.Servers[0]+":53")
+	if err != nil {
+		return nil, err
+	}
+	if r.Rcode != dns.RcodeSuccess {
+		return nil, errors.New("bad rcode: " + strconv.Itoa(r.Rcode))
+	}
+
+	var result []NSRecord
+	for _, a := range r.Answer {
+		if ar, ok := a.(*dns.A); ok {
+			// spew.Dump(ar)
+			val := netaddr.Inet{Inet: ar.A}
+			result = append(result, NSRecord{Name: a.Header().Name, Kind: "A", Value: val})
+			// fmt.Printf("%s\n", mx.String())
+		}
+	}
+
+	return result, nil
+}

checks/cdns/config.go

@@ -0,0 +1,39 @@
+package cdns
+
+import (
+	"time"
+
+	"github.com/miekg/dns"
+)
+
+// DNS check configuration constants.
+const (
+	// Timeout is the DNS query timeout in seconds.
+	Timeout        float64 = float64(1.5)
+	MaxTrials      uint    = 3
+	MaxNameservers uint    = 20
+	MaxAddresses   uint    = 10
+	EdnsbufferSize uint16  = 4096
+)
+
+var (
+	conf           *dns.ClientConfig
+	debug          = false
+	maxTrials      = 3
+	v4only         = true
+	v6only         = false
+	bufsize        = EdnsbufferSize
+	timeout        = time.Duration(float64(Timeout) * float64(time.Second))
+	noedns         = false
+	recursion      = false
+	tcp            = false
+	noauthrequired = false
+	nodnssec       = false
+)
+
+func init() {
+	conf = &dns.ClientConfig{
+		Servers: []string{"8.8.8.8", "1.1.1.1", "77.88.8.8"},
+		Port:    "53",
+	}
+}

checks/cdns/dns.go

@@ -0,0 +1,234 @@
+// Source: https://github.com/bortzmeyer/check-soa/blob/master/check-soa.go
+// 2-Clause BSD License:  Copyright (c) 2012, Stephane Bortzmeyer All rights reserved.
+// A simple program to have rapidly an idea of the health of a DNS
+// zone. It queries each name server of the zone for the SOA record and
+// displays the value of the serial number for each server.
+//
+// Stephane Bortzmeyer <bortzmeyer@nic.fr>
+// Heavily modified for RSMon
+
+package cdns
+
+import (
+	"errors"
+	"fmt"
+	"net"
+	"strings"
+	"time"
+
+	"github.com/miekg/dns"
+	"github.com/weppos/publicsuffix-go/publicsuffix"
+
+	"rsgit.ru/rsmon/rsmon/app/models"
+)
+
+const (
+	stateOK   = "OK"
+	stateERR  = "ERR"
+	stateWARN = "WARN"
+)
+
+var localhost net.IP
+
+func init() {
+	localhost = net.ParseIP("127.0.0.1")
+}
+
+// Perform checks if domain is resolvable via it's DNS servers
+func Perform(c *models.Check) *Result {
+	result := &Result{}
+	result.State = "FAIL"
+	host := c.Monitor.Host
+
+	if host == "" {
+		result.State = "FAIL"
+		result.Error = errors.New("empty host name")
+		return result
+	}
+
+	start := time.Now()
+	// log.Println("run dns:", host)
+	if host == "localhost" || strings.HasPrefix(host, "localhost:") {
+		result.Warnings = append(result.Warnings, "DNS check not possible for localhost, please disable")
+		return result
+	}
+	addr := net.ParseIP(host)
+	if addr != nil {
+		result.Warnings = append(result.Warnings, "DNS check not possible for ip address, please disable")
+		return result
+	}
+
+	zname, err := publicsuffix.Domain(host)
+	if err != nil {
+		result.Warnings = append(result.Warnings, "Failed to get public suffix: "+err.Error())
+		zname = host
+	}
+	// if zname != host {
+	// result.Infos = append(result.Infos, "not top level domain, running NS check for "+zname)
+	// }
+
+	zone := dns.Fqdn(zname)
+	nsChan := make(chan DNSreply)
+	// log.Println(zone)
+	go localQuery(nsChan, zone, dns.TypeNS)
+	nsResult := <-nsChan
+	if nsResult.r == nil {
+		result.State = stateERR
+		result.Error = fmt.Errorf("cannot retrieve the list of name servers for %s: %s", zone, nsResult.err)
+		return result
+	}
+	if nsResult.r.Rcode == dns.RcodeNameError {
+		result.State = stateERR
+		result.Error = fmt.Errorf("no such domain %s", zone)
+		return result
+	}
+	// spew.Dump(nsResult)
+
+	nslist := make(map[string]nameServer, 0)
+	for i := range nsResult.r.Answer {
+		ans := nsResult.r.Answer[i]
+		if ns, ok := ans.(*dns.NS); ok {
+			name := ns.Ns
+			nslist[name] = nameServer{name: name, ips: make([]string, MaxAddresses)}
+		}
+	}
+
+	// spew.Dump(nslist)
+
+	numNS, numNSaddr, success, results := masterTask(zone, nslist)
+	if success {
+		result.State = stateOK
+	} else {
+		result.State = stateERR
+	}
+
+	if numNS == 0 {
+		result.State = stateERR
+		result.Error = fmt.Errorf("no NS records for zone \"%s\"", zone)
+		return result
+	}
+	if numNSaddr == 0 {
+		result.State = stateERR
+		result.Error = fmt.Errorf("no IP addresses for name servers of %s", zone)
+		return result
+	}
+
+	gallOK := true
+	ganyOK := false
+	failedNS := []string{}
+
+	lzone := dns.Fqdn(host)
+
+	for _, rzt := range results { //nolint:gocritic // range copy is acceptable here
+		// spew.Dump(rzt)
+
+		allOK := true
+		anyOK := false
+		ns := NSServer{Name: rzt.name}
+		for i := 0; i < len(rzt.ips); i++ {
+			ip := NSIP{
+				ResponseTime: rzt.rtts[i],
+				IP:           rzt.ips[i],
+			}
+			if rzt.success[i] {
+				anyOK = true
+				ganyOK = true
+				ip.State = stateOK
+				ip.Serial = rzt.serial[i]
+			} else {
+				allOK = false
+				gallOK = false
+				ip.State = stateERR
+				ip.Error = errors.New(rzt.errMsg[i])
+				failedNS = append(failedNS, rzt.name)
+				// spew.Dump(rzt)
+			}
+			ns.NSIPs = append(ns.NSIPs, ip)
+
+			if result.State == stateOK {
+				// spew.Dump(ns)
+				// log.Println("fetching records for", lzone, "from", ns.NSIPs[0].IP)
+				ns.Response, err = FetchARecords(lzone, ns.NSIPs[0].IP)
+				if err != nil {
+					ns.State = stateERR
+					ns.Error = err
+					gallOK = false
+					failedNS = append(failedNS, rzt.name+"/"+ns.NSIPs[0].IP)
+					// log.Println("failed:", err)
+				}
+			}
+		}
+
+		if len(rzt.ips) == 0 {
+			ns.State = stateERR
+			ns.Error = errors.New(rzt.globalErrMsg)
+			failedNS = append(failedNS, rzt.name+"/no ip for dns server")
+			gallOK = false
+		} else {
+			if allOK {
+				ns.State = stateOK
+			} else {
+				if anyOK {
+					ns.State = "WARN"
+					// log.Println("failed NS")
+					// spew.Dump(ns)
+					if ns.Error == nil {
+						ns.Error = errors.New("some servers failed")
+					}
+				} else {
+					ns.State = stateERR
+					if ns.Error == nil {
+						ns.Error = errors.New("all servers failed")
+					}
+				}
+			}
+		}
+
+		result.NSServers = append(result.NSServers, ns)
+	}
+
+	if gallOK {
+		result.State = stateOK
+	} else {
+		if ganyOK {
+			result.State = stateWARN
+			result.Warnings = append(result.Warnings, "some servers failed: "+strings.Join(failedNS, ","))
+		} else {
+			result.State = stateERR
+			if result.Error == nil {
+				result.Error = errors.New("all servers failed")
+			}
+		}
+	}
+
+	for _, ni := range result.NSServers {
+		// log.Println(ni.Name)
+		for _, r := range ni.Response {
+			ip := r.Value.Inet
+			if ip.Equal(localhost) {
+				result.State = stateERR
+				err := "resolves to localhost/127.0.0.1"
+				if result.Error == nil {
+					result.Error = errors.New(err)
+				} else if result.Error.Error() != err {
+					result.Warnings = append(result.Warnings, err)
+				}
+			}
+			// log.Println(r.Name, r.Kind, r.Value)
+		}
+	}
+
+	_, maxt := result.Times()
+	if maxt > 2*time.Second {
+		if result.State == stateOK {
+			result.State = stateWARN
+		}
+		result.Warnings = append(result.Warnings, "slow")
+	}
+
+	result.Duration = time.Since(start)
+
+	// spew.Dump(result)
+
+	return result
+}

checks/cdns/local_query.go

@@ -0,0 +1,67 @@
+package cdns
+
+import (
+	"errors"
+	"fmt"
+	"log"
+	"strings"
+
+	"github.com/miekg/dns"
+)
+
+func localQuery(mychan chan DNSreply, qname string, qtype uint16) {
+	if debug {
+		fmt.Printf("DEBUG: start of DNS request \"%s\" / %d\n", qname, qtype)
+	}
+	var result DNSreply
+	var trials uint
+	result.qname = qname
+	result.qtype = qtype
+	result.r = nil
+	result.err = errors.New("no name server to answer the question")
+	localm := new(dns.Msg)
+	localm.Id = dns.Id()
+	localm.RecursionDesired = true
+	localm.Question = make([]dns.Question, 1)
+	localm.SetEdns0(bufsize, false) // Even if no EDNS requested, see #9 May be we should retry without it if timeout?
+	localc := new(dns.Client)
+	localc.ReadTimeout = timeout
+	localm.Question[0] = dns.Question{Name: qname, Qtype: qtype, Qclass: dns.ClassINET}
+Tests:
+	for trials = 0; trials < uint(maxTrials); trials++ {
+		for serverIndex := range conf.Servers {
+			server := conf.Servers[serverIndex]
+			result.nameserver = server
+			// Brackets around the server address are necessary for IPv6 name servers
+			// Brackets required for IPv6; do not use net.JoinHostPort (see check-soa commit 3e4edb1)
+			r, rtt, err := localc.Exchange(localm, "["+server+"]:"+conf.Port)
+			if r == nil {
+				result.r = nil
+				result.err = err
+				log.Println(err.Error())
+				if strings.Contains(err.Error(), "timeout") {
+					// Try another resolver
+					continue
+				}
+				// We give in
+				break Tests
+			}
+			result.rtt = rtt
+			if r.Rcode == dns.RcodeSuccess {
+				// TODO: NODATA (NOERROR/ANSWER=0) are silently ignored (e.g. name exists but no IP address)
+				// TODO: for rcodes like SERVFAIL, trying another resolver could make sense
+				result.r = r
+				result.err = nil
+				break Tests
+			}
+			// All the other codes are errors
+			result.r = r
+			result.err = errors.New(dns.RcodeToString[r.Rcode])
+			break Tests
+		}
+	}
+	if debug {
+		fmt.Printf("DEBUG: end of DNS request \"%s\" / %d\n", qname, qtype)
+	}
+	mychan <- result
+}

checks/cdns/master_task.go

@@ -0,0 +1,123 @@
+package cdns
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/miekg/dns"
+)
+
+// Results provides functionality.
+type Results map[string]nameServer
+
+func masterTask(zone string, nameservers map[string]nameServer) (uint, uint, bool, Results) {
+	var numRequests uint
+	success := true
+	addressChannel := make(chan DNSreply)
+	soaChannel := make(chan SOAreply)
+	numNS := uint(0)
+	numAddrNS := uint(0)
+	results := make(Results)
+	for name := range nameservers {
+		if !v6only {
+			go localQuery(addressChannel, name, dns.TypeA)
+		}
+		if !v4only {
+			go localQuery(addressChannel, name, dns.TypeAAAA)
+		}
+		numNS++
+	}
+	if v6only || v4only {
+		numRequests = numNS
+	} else {
+		numRequests = numNS * 2
+	}
+	for i := uint(0); i < numRequests; i++ {
+		addrResult := <-addressChannel
+		addrFamily := "IPv6"
+		if addrResult.qtype == dns.TypeA {
+			addrFamily = "IPv4"
+		}
+		if addrResult.r == nil {
+			// TODO We may have different globalErrMsg is it
+			// works with IPv4 but not IPv6 (it should not happen but it does)
+			nameservers[addrResult.qname] = nameServer{
+				name:         addrResult.qname,
+				ips:          nil,
+				globalErrMsg: fmt.Sprintf("Cannot get the %s address: %s", addrFamily, addrResult.err),
+			}
+			success = false
+		} else {
+			if addrResult.r.Rcode != dns.RcodeSuccess {
+				nameservers[addrResult.qname] = nameServer{
+					name:         addrResult.qname,
+					ips:          nil,
+					globalErrMsg: fmt.Sprintf("Cannot get the %s address: %s", addrFamily, dns.RcodeToString[addrResult.r.Rcode]),
+				}
+				success = false
+			} else {
+				for j := range addrResult.r.Answer {
+					ansa := addrResult.r.Answer[j]
+					var ns string
+					switch a := ansa.(type) {
+					case *dns.A:
+						ns = a.A.String()
+						existing := nameservers[addrResult.qname]
+						nameservers[addrResult.qname] = nameServer{name: addrResult.qname, ips: append(existing.ips, ns)}
+						numAddrNS++
+						go soaQuery(soaChannel, zone, addrResult.qname, ns)
+					case *dns.AAAA:
+						ns = a.AAAA.String()
+						existing2 := nameservers[addrResult.qname]
+						nameservers[addrResult.qname] = nameServer{name: addrResult.qname, ips: append(existing2.ips, ns)}
+						numAddrNS++
+						go soaQuery(soaChannel, zone, addrResult.qname, ns)
+					}
+				}
+			}
+		}
+	}
+	for i := uint(0); i < numAddrNS; i++ {
+		if debug {
+			fmt.Printf("DEBUG Getting result for ns #%d/%d\n", i+1, numAddrNS)
+		}
+		soaResult := <-soaChannel
+		_, present := results[soaResult.name]
+		if !present {
+			results[soaResult.name] = nameServer{
+				name:    soaResult.name,
+				ips:     make([]string, 0),
+				success: make([]bool, 0),
+				errMsg:  make([]string, 0),
+				serial:  make([]uint32, 0),
+				rtts:    make([]time.Duration, 0),
+			}
+		}
+		if !soaResult.retrieved {
+			results[soaResult.name] = nameServer{
+				name:    soaResult.name,
+				ips:     append(results[soaResult.name].ips, soaResult.address),
+				success: append(results[soaResult.name].success, false),
+				errMsg:  append(results[soaResult.name].errMsg, soaResult.msg),
+				serial:  append(results[soaResult.name].serial, 0),
+				rtts:    append(results[soaResult.name].rtts, soaResult.rtt),
+			}
+			success = false
+		} else {
+			results[soaResult.name] = nameServer{
+				name:    soaResult.name,
+				ips:     append(results[soaResult.name].ips, soaResult.address),
+				success: append(results[soaResult.name].success, true),
+				errMsg:  append(results[soaResult.name].errMsg, ""),
+				serial:  append(results[soaResult.name].serial, soaResult.serial),
+				rtts:    append(results[soaResult.name].rtts, soaResult.rtt),
+			}
+		}
+	}
+	for name := range nameservers {
+		if nameservers[name].ips == nil {
+			results[name] = nameservers[name]
+		}
+	}
+	return numNS, numAddrNS, success, results
+}

checks/cdns/result.go

@@ -0,0 +1,109 @@
+package cdns
+
+import (
+	"strconv"
+	"strings"
+	"time"
+
+	"rsgit.ru/rsmon/rsmon/app/models"
+	"rsgit.ru/rsmon/rsmon/internal/checkresult"
+	"rsgit.ru/rsmon/rsmon/internal/netaddr"
+)
+
+// NSIP holds the result of querying a single nameserver IP.
+type NSIP struct {
+	State        string
+	IP           string
+	ResponseTime time.Duration
+	Serial       uint32
+	Error        error
+}
+
+// NSRecord holds a single DNS record from a nameserver response.
+type NSRecord struct {
+	Name  string
+	Kind  string
+	Value netaddr.Inet
+}
+
+// NSServer holds the result of querying a single nameserver.
+type NSServer struct {
+	State    string
+	Name     string
+	Error    error
+	NSIPs    []NSIP
+	Response []NSRecord
+}
+
+// Result holds the full DNS check result.
+type Result struct {
+	checkresult.CheckResult
+	NSServers []NSServer
+}
+
+// Servers provides functionality.
+func (r *Result) Servers() []string {
+	ret := make([]string, 0, len(r.NSServers))
+	for _, s := range r.NSServers {
+		ret = append(ret, s.Name+"-"+s.State)
+	}
+	return ret
+}
+
+// ServersOK provides functionality.
+func (r *Result) ServersOK() int {
+	ok := 0
+	for _, s := range r.NSServers {
+		if s.State == "OK" {
+			ok++
+		}
+	}
+	return ok
+}
+
+// Times provides functionality.
+func (r *Result) Times() (time.Duration, time.Duration) {
+	minTime := time.Hour
+	maxTime := time.Duration(0)
+	for _, s := range r.NSServers {
+		for _, i := range s.NSIPs {
+			if i.ResponseTime > maxTime {
+				maxTime = i.ResponseTime
+			}
+			if i.ResponseTime < minTime {
+				minTime = i.ResponseTime
+			}
+		}
+	}
+	return minTime, maxTime
+}
+
+// InfluxFields provides functionality.
+func (r *Result) InfluxFields() map[string]interface{} {
+	ret := make(map[string]interface{}, 0)
+
+	_, maxTime := r.Times()
+	ret["took"] = int64(maxTime / time.Millisecond)
+
+	return ret
+}
+
+// InfluxTags provides functionality.
+func (r *Result) InfluxTags(c models.Check) map[string]string { //nolint:gocritic // hugeParam: accepted for interface compatibility
+	ret := make(map[string]string, 0)
+	ret["check"] = strconv.FormatInt(c.ID, 10)
+	ret["state"] = c.State
+	minTime, _ := r.Times()
+	ret["min_took"] = strconv.FormatInt(int64(minTime/time.Millisecond), 10)
+
+	ret["state"] = r.State
+	total := len(r.NSServers)
+	oks := r.ServersOK()
+	ret["warnings"] = strings.Join(r.Warnings, ",")
+
+	ret["nservers"] = strconv.Itoa(total)
+	ret["servers"] = strings.Join(r.Servers(), ",")
+	ret["servers_ok"] = strconv.Itoa(oks)
+	ret["servers_failed"] = strconv.Itoa(total - oks)
+	return ret
+}

checks/cdns/soa_query.go

@@ -0,0 +1,82 @@
+package cdns
+
+import (
+	"fmt"
+	"net"
+
+	"github.com/miekg/dns"
+)
+
+func soaQuery(mychan chan SOAreply, zone string, name string, server string) {
+	var result SOAreply
+	var trials uint
+	result.retrieved = false
+	result.name = name
+	result.address = server
+	result.msg = "UNKNOWN"
+	m := new(dns.Msg)
+	if !noedns {
+		m.SetEdns0(bufsize, !nodnssec)
+	}
+	m.Id = dns.Id()
+	if recursion {
+		m.RecursionDesired = true
+	} else {
+		m.RecursionDesired = false
+	}
+	m.Question = make([]dns.Question, 1)
+	c := new(dns.Client)
+	c.ReadTimeout = timeout // Seems ignored for TCP?
+	if tcp {
+		c.Net = "tcp"
+	}
+	m.Question[0] = dns.Question{Name: zone, Qtype: dns.TypeSOA, Qclass: dns.ClassINET}
+	nsAddressPort := net.JoinHostPort(server, "53")
+	if debug {
+		fmt.Printf("DEBUG Querying SOA from %s\n", nsAddressPort)
+	}
+	for trials = 0; trials < uint(maxTrials); trials++ {
+		soa, rtt, err := c.Exchange(m, nsAddressPort)
+		if soa == nil {
+			result.rtt = 0
+			result.msg = err.Error()
+		} else {
+			result.rtt = rtt
+			if soa.Rcode != dns.RcodeSuccess {
+				result.msg = dns.RcodeToString[soa.Rcode]
+				break
+			}
+			if len(soa.Answer) == 0 { /* May happen if the server is a recursor, not authoritative, since we query with RD=0 */
+				result.msg = "0 answer"
+				break
+			} else { //nolint:revive // complex nested structure
+				gotSoa := false
+				for _, rsoa := range soa.Answer {
+					switch r := rsoa.(type) {
+					case *dns.SOA:
+						if noauthrequired || soa.Authoritative {
+							result.retrieved = true
+							result.serial = r.Serial
+							result.msg = "OK"
+						} else {
+							result.msg = "Not authoritative"
+						}
+						gotSoa = true
+					case *dns.CNAME: /* Bad practice but common */
+						result.msg = "Apparently not a zone but an alias"
+					case *dns.RRSIG:
+						/* Ignore them. See bug #8 */
+					default:
+						// TODO: a name server can send us other RR types.
+						result.msg = fmt.Sprintf("Internal error when processing %s, unexpected record type\n", rsoa)
+					}
+				}
+				if !gotSoa {
+					result.msg = "No SOA record in reply"
+				}
+				break
+			}
+		}
+	}
+	mychan <- result
+}

checks/cdns/types.go

@@ -0,0 +1,37 @@
+package cdns
+
+import (
+	"time"
+
+	"github.com/miekg/dns"
+)
+
+// DNSreply provides functionality.
+type DNSreply struct {
+	qname      string
+	qtype      uint16
+	r          *dns.Msg
+	err        error
+	nameserver string
+	rtt        time.Duration
+}
+
+// SOAreply provides functionality.
+type SOAreply struct {
+	name      string
+	address   string
+	serial    uint32
+	retrieved bool
+	msg       string
+	rtt       time.Duration
+}
+
+type nameServer struct {
+	name         string
+	ips          []string
+	globalErrMsg string
+	success      []bool
+	errMsg       []string
+	serial       []uint32
+	rtts         []time.Duration
+}