diff --git a/app/models/monitor.go b/app/models/monitor.go index d44d2cb..cd170d3 100644 --- a/app/models/monitor.go +++ b/app/models/monitor.go @@ -297,7 +297,8 @@ var mutex sync.Mutex // checkSeverityRank assigns an ordinal to each check state so the monitor // aggregator can pick the highest-severity child deterministically. -// Severity order is FAIL > ERR > DEGRADED > WARN > OK — see docs/todo.md +// Severity order is FAIL > ERR > DEGRADED > WARN > OK. FAIL means the +// worker could not run a check; it remains visible but never opens alerts. // Phase 3 for the rationale (DEGRADED = partial regional failure, sits // between OK and ERR). Unknown states (UNK, empty, ...) rank 0 so any // real check state takes precedence over them. @@ -341,7 +342,7 @@ func (m *Monitor) UpdateStatusFromChecks() { // inlined three if-statements with non-obvious precedence (a WARN that // appeared AFTER an ERR in the iteration would never downgrade back, // but a FAIL after ERR would silently get clobbered). Using a single - // severity rank keeps the rule FAIL > ERR > DEGRADED > WARN > OK + // severity rank keeps the overall health state deterministic. // independent of slice ordering — the same rule Phase 3 introduces // for DEGRADED, applied uniformly to the existing states too. bestRank := checkSeverityRank(stateOK) @@ -383,7 +384,7 @@ func (m *Monitor) UpdateStatusFromChecks() { // log.Println("state", m.State, "active event:", evt.ID) for _, check := range checks { //nolint:gocritic // range copy is acceptable here - if check.State == stateERR || check.State == stateFail { + if check.State == stateERR { evt.ChecksDown = append(evt.ChecksDown, check.Kind) evt.Checks = append(evt.Checks, check) if check.Error != nil { @@ -419,7 +420,7 @@ func (m *Monitor) UpdateStatusFromChecks() { return } } - case stateERR, stateFail: + case stateERR: if evt.ID == 0 { tn := time.Now() evt.StartTime = &tn @@ -500,7 +501,7 @@ func (m *Monitor) syncStatusPageIncidentsTx(tx *gorm.DB, event *Event) error { page := &pages[i] var incident StatusPageIncident err := tx.Where("status_page_id = ? AND event_id = ?", page.ID, event.ID).First(&incident).Error - if m.State == stateERR || m.State == "FAIL" { + if m.State == stateERR { if err != nil { // The database uniqueness constraint makes concurrent state updates idempotent. incident = StatusPageIncident{StatusPageID: page.ID, EventID: &event.ID, Title: m.GetLabel() + " is unavailable", BodyMD: event.Reason, Severity: StatusPageIncidentSeverityCrit, StartedAt: time.Now()} diff --git a/checks/cdns/config.go b/checks/cdns/config.go index 8f5a987..6348542 100644 --- a/checks/cdns/config.go +++ b/checks/cdns/config.go @@ -1,6 +1,9 @@ package cdns import ( + "net" + "os" + "strings" "time" "github.com/miekg/dns" @@ -33,7 +36,27 @@ var ( func init() { conf = &dns.ClientConfig{ - Servers: []string{"8.8.8.8", "1.1.1.1", "77.88.8.8"}, + Servers: resolversFromEnv(), Port: "53", } } + +func resolversFromEnv() []string { + const defaults = "188.93.17.19,188.93.16.19,82.117.245.122" + resolvers := strings.Split(os.Getenv("RSMON_DNS_RESOLVERS"), ",") + if strings.TrimSpace(os.Getenv("RSMON_DNS_RESOLVERS")) == "" { + resolvers = strings.Split(defaults, ",") + } + + valid := make([]string, 0, len(resolvers)) + for _, resolver := range resolvers { + resolver = strings.TrimSpace(resolver) + if net.ParseIP(resolver) != nil { + valid = append(valid, resolver) + } + } + if len(valid) == 0 { + return strings.Split(defaults, ",") + } + return valid +} diff --git a/checks/cdns/config_test.go b/checks/cdns/config_test.go new file mode 100644 index 0000000..e364155 --- /dev/null +++ b/checks/cdns/config_test.go @@ -0,0 +1,26 @@ +package cdns + +import ( + "reflect" + "testing" +) + +func TestResolversFromEnv(t *testing.T) { + t.Setenv("RSMON_DNS_RESOLVERS", " 188.93.17.19, 2001:db8::53, invalid ") + + got := resolversFromEnv() + want := []string{"188.93.17.19", "2001:db8::53"} + if !reflect.DeepEqual(got, want) { + t.Fatalf("resolversFromEnv() = %v, want %v", got, want) + } +} + +func TestResolversFromEnvFallsBackToDefaults(t *testing.T) { + t.Setenv("RSMON_DNS_RESOLVERS", "not-an-address") + + got := resolversFromEnv() + want := []string{"188.93.17.19", "188.93.16.19", "82.117.245.122"} + if !reflect.DeepEqual(got, want) { + t.Fatalf("resolversFromEnv() = %v, want %v", got, want) + } +} diff --git a/checks/cdns/dns.go b/checks/cdns/dns.go index ea08e9b..3baeae1 100644 --- a/checks/cdns/dns.go +++ b/checks/cdns/dns.go @@ -73,7 +73,9 @@ func Perform(c *models.Check) *Result { go localQuery(nsChan, zone, dns.TypeNS) nsResult := <-nsChan if nsResult.r == nil { - result.State = stateERR + // All recursive resolvers were unreachable, so this worker could not + // establish whether the domain is healthy. Do not open an outage event. + result.State = "FAIL" result.Error = fmt.Errorf("cannot retrieve the list of name servers for %s: %s", zone, nsResult.err) return result } diff --git a/checks/cdns/local_query.go b/checks/cdns/local_query.go index f8220cb..b5d5921 100644 --- a/checks/cdns/local_query.go +++ b/checks/cdns/local_query.go @@ -4,7 +4,6 @@ import ( "errors" "fmt" "log" - "strings" "github.com/miekg/dns" ) @@ -14,7 +13,6 @@ func localQuery(mychan chan DNSreply, qname string, qtype uint16) { fmt.Printf("DEBUG: start of DNS request \"%s\" / %d\n", qname, qtype) } var result DNSreply - var trials uint result.qname = qname result.qtype = qtype result.r = nil @@ -24,41 +22,29 @@ func localQuery(mychan chan DNSreply, qname string, qtype uint16) { localm.RecursionDesired = true localm.Question = make([]dns.Question, 1) localm.SetEdns0(bufsize, false) // Even if no EDNS requested, see #9 May be we should retry without it if timeout? - localc := new(dns.Client) - localc.ReadTimeout = timeout localm.Question[0] = dns.Question{Name: qname, Qtype: qtype, Qclass: dns.ClassINET} -Tests: - for trials = 0; trials < uint(maxTrials); trials++ { - for serverIndex := range conf.Servers { - server := conf.Servers[serverIndex] - result.nameserver = server - // Brackets around the server address are necessary for IPv6 name servers - // Brackets required for IPv6; do not use net.JoinHostPort (see check-soa commit 3e4edb1) - r, rtt, err := localc.Exchange(localm, "["+server+"]:"+conf.Port) - if r == nil { - result.r = nil - result.err = err - log.Println(err.Error()) - if strings.Contains(err.Error(), "timeout") { - // Try another resolver - continue - } - // We give in - break Tests + for _, server := range conf.Servers { + result.nameserver = server + localc := &dns.Client{ReadTimeout: timeout} + // Brackets are required for IPv6; do not use net.JoinHostPort (see check-soa commit 3e4edb1). + r, rtt, err := localc.Exchange(localm, "["+server+"]:"+conf.Port) + if err != nil || r == nil { + result.r = nil + result.err = err + if err != nil { + log.Println(err) } - result.rtt = rtt - if r.Rcode == dns.RcodeSuccess { - // TODO: NODATA (NOERROR/ANSWER=0) are silently ignored (e.g. name exists but no IP address) - // TODO: for rcodes like SERVFAIL, trying another resolver could make sense - result.r = r - result.err = nil - break Tests - } - // All the other codes are errors - result.r = r - result.err = errors.New(dns.RcodeToString[r.Rcode]) - break Tests + continue } + result.rtt = rtt + if r.Rcode == dns.RcodeSuccess { + result.r = r + result.err = nil + break + } + result.r = r + result.err = errors.New(dns.RcodeToString[r.Rcode]) + break } if debug { fmt.Printf("DEBUG: end of DNS request \"%s\" / %d\n", qname, qtype) diff --git a/packaging/systemd/worker.env.example b/packaging/systemd/worker.env.example index 66ffcc7..94b7501 100644 --- a/packaging/systemd/worker.env.example +++ b/packaging/systemd/worker.env.example @@ -14,3 +14,6 @@ WORKER_CLUSTER_PEERS= WORKER_CLUSTER_DATA_DIR=/var/lib/rsmon-worker/cluster WORKER_CLUSTER_BOOTSTRAP=false WORKER_RELEASE_URL= +# Comma-separated IPv4/IPv6 recursive resolvers used by DNS checks. Empty +# uses 188.93.17.19, 188.93.16.19, and our IPv4-only resolver 82.117.245.122. +RSMON_DNS_RESOLVERS=188.93.17.19,188.93.16.19,82.117.245.122