fix(dns): retry configurable resolvers

Treat resolver transport failures as inconclusive checks so they do not open outage alerts.

app/models/monitor.go

@@ -297,7 +297,8 @@ var mutex sync.Mutex
 
 // checkSeverityRank assigns an ordinal to each check state so the monitor
 // aggregator can pick the highest-severity child deterministically.
-// Severity order is FAIL > ERR > DEGRADED > WARN > OK — see docs/todo.md
+// Severity order is FAIL > ERR > DEGRADED > WARN > OK. FAIL means the
+// worker could not run a check; it remains visible but never opens alerts.
 // Phase 3 for the rationale (DEGRADED = partial regional failure, sits
 // between OK and ERR). Unknown states (UNK, empty, ...) rank 0 so any
 // real check state takes precedence over them.
@@ -341,7 +342,7 @@ func (m *Monitor) UpdateStatusFromChecks() {
 	// inlined three if-statements with non-obvious precedence (a WARN that
 	// appeared AFTER an ERR in the iteration would never downgrade back,
 	// but a FAIL after ERR would silently get clobbered). Using a single
-	// severity rank keeps the rule FAIL > ERR > DEGRADED > WARN > OK
+	// severity rank keeps the overall health state deterministic.
 	// independent of slice ordering — the same rule Phase 3 introduces
 	// for DEGRADED, applied uniformly to the existing states too.
 	bestRank := checkSeverityRank(stateOK)
@@ -383,7 +384,7 @@ func (m *Monitor) UpdateStatusFromChecks() {
 	// log.Println("state", m.State, "active event:", evt.ID)
 
 	for _, check := range checks { //nolint:gocritic // range copy is acceptable here
-		if check.State == stateERR || check.State == stateFail {
+		if check.State == stateERR {
 			evt.ChecksDown = append(evt.ChecksDown, check.Kind)
 			evt.Checks = append(evt.Checks, check)
 			if check.Error != nil {
@@ -419,7 +420,7 @@ func (m *Monitor) UpdateStatusFromChecks() {
 				return
 			}
 		}
-	case stateERR, stateFail:
+	case stateERR:
 		if evt.ID == 0 {
 			tn := time.Now()
 			evt.StartTime = &tn
@@ -500,7 +501,7 @@ func (m *Monitor) syncStatusPageIncidentsTx(tx *gorm.DB, event *Event) error {
 		page := &pages[i]
 		var incident StatusPageIncident
 		err := tx.Where("status_page_id = ? AND event_id = ?", page.ID, event.ID).First(&incident).Error
-		if m.State == stateERR || m.State == "FAIL" {
+		if m.State == stateERR {
 			if err != nil {
 				// The database uniqueness constraint makes concurrent state updates idempotent.
 				incident = StatusPageIncident{StatusPageID: page.ID, EventID: &event.ID, Title: m.GetLabel() + " is unavailable", BodyMD: event.Reason, Severity: StatusPageIncidentSeverityCrit, StartedAt: time.Now()}

checks/cdns/config.go

@@ -1,6 +1,9 @@
 package cdns
 
 import (
+	"net"
+	"os"
+	"strings"
 	"time"
 
 	"github.com/miekg/dns"
@@ -33,7 +36,27 @@ var (
 
 func init() {
 	conf = &dns.ClientConfig{
-		Servers: []string{"8.8.8.8", "1.1.1.1", "77.88.8.8"},
+		Servers: resolversFromEnv(),
 		Port:    "53",
 	}
 }
+
+func resolversFromEnv() []string {
+	const defaults = "188.93.17.19,188.93.16.19,82.117.245.122"
+	resolvers := strings.Split(os.Getenv("RSMON_DNS_RESOLVERS"), ",")
+	if strings.TrimSpace(os.Getenv("RSMON_DNS_RESOLVERS")) == "" {
+		resolvers = strings.Split(defaults, ",")
+	}
+
+	valid := make([]string, 0, len(resolvers))
+	for _, resolver := range resolvers {
+		resolver = strings.TrimSpace(resolver)
+		if net.ParseIP(resolver) != nil {
+			valid = append(valid, resolver)
+		}
+	}
+	if len(valid) == 0 {
+		return strings.Split(defaults, ",")
+	}
+	return valid
+}

checks/cdns/config_test.go

@@ -0,0 +1,26 @@
+package cdns
+
+import (
+	"reflect"
+	"testing"
+)
+
+func TestResolversFromEnv(t *testing.T) {
+	t.Setenv("RSMON_DNS_RESOLVERS", " 188.93.17.19, 2001:db8::53, invalid ")
+
+	got := resolversFromEnv()
+	want := []string{"188.93.17.19", "2001:db8::53"}
+	if !reflect.DeepEqual(got, want) {
+		t.Fatalf("resolversFromEnv() = %v, want %v", got, want)
+	}
+}
+
+func TestResolversFromEnvFallsBackToDefaults(t *testing.T) {
+	t.Setenv("RSMON_DNS_RESOLVERS", "not-an-address")
+
+	got := resolversFromEnv()
+	want := []string{"188.93.17.19", "188.93.16.19", "82.117.245.122"}
+	if !reflect.DeepEqual(got, want) {
+		t.Fatalf("resolversFromEnv() = %v, want %v", got, want)
+	}
+}

checks/cdns/dns.go

@@ -73,7 +73,9 @@ func Perform(c *models.Check) *Result {
 	go localQuery(nsChan, zone, dns.TypeNS)
 	nsResult := <-nsChan
 	if nsResult.r == nil {
-		result.State = stateERR
+		// All recursive resolvers were unreachable, so this worker could not
+		// establish whether the domain is healthy. Do not open an outage event.
+		result.State = "FAIL"
 		result.Error = fmt.Errorf("cannot retrieve the list of name servers for %s: %s", zone, nsResult.err)
 		return result
 	}

checks/cdns/local_query.go

@@ -4,7 +4,6 @@ import (
 	"errors"
 	"fmt"
 	"log"
-	"strings"
 
 	"github.com/miekg/dns"
 )
@@ -14,7 +13,6 @@ func localQuery(mychan chan DNSreply, qname string, qtype uint16) {
 		fmt.Printf("DEBUG: start of DNS request \"%s\" / %d\n", qname, qtype)
 	}
 	var result DNSreply
-	var trials uint
 	result.qname = qname
 	result.qtype = qtype
 	result.r = nil
@@ -24,41 +22,29 @@ func localQuery(mychan chan DNSreply, qname string, qtype uint16) {
 	localm.RecursionDesired = true
 	localm.Question = make([]dns.Question, 1)
 	localm.SetEdns0(bufsize, false) // Even if no EDNS requested, see #9 May be we should retry without it if timeout?
-	localc := new(dns.Client)
-	localc.ReadTimeout = timeout
 	localm.Question[0] = dns.Question{Name: qname, Qtype: qtype, Qclass: dns.ClassINET}
-Tests:
+	for _, server := range conf.Servers {
-	for trials = 0; trials < uint(maxTrials); trials++ {
+		result.nameserver = server
-		for serverIndex := range conf.Servers {
+		localc := &dns.Client{ReadTimeout: timeout}
-			server := conf.Servers[serverIndex]
+		// Brackets are required for IPv6; do not use net.JoinHostPort (see check-soa commit 3e4edb1).
-			result.nameserver = server
+		r, rtt, err := localc.Exchange(localm, "["+server+"]:"+conf.Port)
-			// Brackets around the server address are necessary for IPv6 name servers
+		if err != nil || r == nil {
-			// Brackets required for IPv6; do not use net.JoinHostPort (see check-soa commit 3e4edb1)
+			result.r = nil
-			r, rtt, err := localc.Exchange(localm, "["+server+"]:"+conf.Port)
+			result.err = err
-			if r == nil {
+			if err != nil {
-				result.r = nil
+				log.Println(err)
-				result.err = err
-				log.Println(err.Error())
-				if strings.Contains(err.Error(), "timeout") {
-					// Try another resolver
-					continue
-				}
-				// We give in
-				break Tests
 			}
-			result.rtt = rtt
+			continue
-			if r.Rcode == dns.RcodeSuccess {
-				// TODO: NODATA (NOERROR/ANSWER=0) are silently ignored (e.g. name exists but no IP address)
-				// TODO: for rcodes like SERVFAIL, trying another resolver could make sense
-				result.r = r
-				result.err = nil
-				break Tests
-			}
-			// All the other codes are errors
-			result.r = r
-			result.err = errors.New(dns.RcodeToString[r.Rcode])
-			break Tests
 		}
+		result.rtt = rtt
+		if r.Rcode == dns.RcodeSuccess {
+			result.r = r
+			result.err = nil
+			break
+		}
+		result.r = r
+		result.err = errors.New(dns.RcodeToString[r.Rcode])
+		break
 	}
 	if debug {
 		fmt.Printf("DEBUG: end of DNS request \"%s\" / %d\n", qname, qtype)

packaging/systemd/worker.env.example

@@ -14,3 +14,6 @@ WORKER_CLUSTER_PEERS=
 WORKER_CLUSTER_DATA_DIR=/var/lib/rsmon-worker/cluster
 WORKER_CLUSTER_BOOTSTRAP=false
 WORKER_RELEASE_URL=
+# Comma-separated IPv4/IPv6 recursive resolvers used by DNS checks. Empty
+# uses 188.93.17.19, 188.93.16.19, and our IPv4-only resolver 82.117.245.122.
+RSMON_DNS_RESOLVERS=188.93.17.19,188.93.16.19,82.117.245.122