fix(dns): retry configurable resolvers
Все проверки выполнены успешно
CI / test (push) Successful in 2m32s
Docker / Build and publish worker image (push) Successful in 20m9s

Treat resolver transport failures as inconclusive checks so they do not open outage alerts.
Этот коммит содержится в:
Gleb Tv
2026-07-16 19:30:02 +03:00
родитель fd1a010e31
Коммит 579a14b403
6 изменённых файлов: 82 добавлений и 41 удалений

Просмотреть файл

@@ -297,7 +297,8 @@ var mutex sync.Mutex
// checkSeverityRank assigns an ordinal to each check state so the monitor
// aggregator can pick the highest-severity child deterministically.
// Severity order is FAIL > ERR > DEGRADED > WARN > OK — see docs/todo.md
// Severity order is FAIL > ERR > DEGRADED > WARN > OK. FAIL means the
// worker could not run a check; it remains visible but never opens alerts.
// Phase 3 for the rationale (DEGRADED = partial regional failure, sits
// between OK and ERR). Unknown states (UNK, empty, ...) rank 0 so any
// real check state takes precedence over them.
@@ -341,7 +342,7 @@ func (m *Monitor) UpdateStatusFromChecks() {
// inlined three if-statements with non-obvious precedence (a WARN that
// appeared AFTER an ERR in the iteration would never downgrade back,
// but a FAIL after ERR would silently get clobbered). Using a single
// severity rank keeps the rule FAIL > ERR > DEGRADED > WARN > OK
// severity rank keeps the overall health state deterministic.
// independent of slice ordering — the same rule Phase 3 introduces
// for DEGRADED, applied uniformly to the existing states too.
bestRank := checkSeverityRank(stateOK)
@@ -383,7 +384,7 @@ func (m *Monitor) UpdateStatusFromChecks() {
// log.Println("state", m.State, "active event:", evt.ID)
for _, check := range checks { //nolint:gocritic // range copy is acceptable here
if check.State == stateERR || check.State == stateFail {
if check.State == stateERR {
evt.ChecksDown = append(evt.ChecksDown, check.Kind)
evt.Checks = append(evt.Checks, check)
if check.Error != nil {
@@ -419,7 +420,7 @@ func (m *Monitor) UpdateStatusFromChecks() {
return
}
}
case stateERR, stateFail:
case stateERR:
if evt.ID == 0 {
tn := time.Now()
evt.StartTime = &tn
@@ -500,7 +501,7 @@ func (m *Monitor) syncStatusPageIncidentsTx(tx *gorm.DB, event *Event) error {
page := &pages[i]
var incident StatusPageIncident
err := tx.Where("status_page_id = ? AND event_id = ?", page.ID, event.ID).First(&incident).Error
if m.State == stateERR || m.State == "FAIL" {
if m.State == stateERR {
if err != nil {
// The database uniqueness constraint makes concurrent state updates idempotent.
incident = StatusPageIncident{StatusPageID: page.ID, EventID: &event.ID, Title: m.GetLabel() + " is unavailable", BodyMD: event.Reason, Severity: StatusPageIncidentSeverityCrit, StartedAt: time.Now()}