collabSupport/services/enrichment/alarmSemantics.js
jmcqueen 21fa8f1436 Clean up SD-WAN alarm rendering + explain overlay vs physical
The wanAlarms check was pasting raw Prisma `info` JSON blobs (nested
`vpn_reasons` arrays with element/site/vpnlink ids) into the chat
message field. On a store with 20 NETWORK_ANYNETLINK_DOWN flaps this
produced a wall of unreadable stringified JSON where the actual
signal ("SD-WAN overlay tunnels are flapping") was lost.

Introduces a shared alarmSemantics module that:
  - Buckets each code into overlay / physical / device / other so
    the check + renderer stay consistent
  - Humanizes codes (NETWORK_ANYNETLINK_DOWN → "SD-WAN overlay
    tunnel down") with a Title-Cased fallback for unknown codes
  - Rolls up (code + severity) tuples so 20 identical alarms show as
    a single line with ×20 and a "just now / Nm / Nh / Nd" age

Rewrites wanAlarms.run() to use those helpers + cross-reference the
site's physical link state so operators aren't left wondering why 20
alarms fired while every metric shows green: overlay flaps get a
"physical WAN paths are all up per Link State" clarifier, and
physical alarms point back at the Link State check. The label loses
its hardcoded "(last 1h)" suffix since the alarm window is now
dynamic (defaults to 24h to match the WAN window).

The follow-up renderer used by /phonestatus imports the same helpers
so the two surfaces cannot drift.

Co-authored-by: Cursor <cursoragent@cursor.com>
2026-07-09 10:19:15 -04:00

234 lines
9.8 KiB
JavaScript
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

// src/services/enrichment/alarmSemantics.js
//
// Shared alarm-code semantics for Prisma SD-WAN events. Two things
// live here so both the check runner (services/voiceDiag/checks/wan
// /wanAlarms.js) and the follow-up renderer (services/renderers/
// wanDiagnosticsRenderer.js) present the same story:
//
// 1. categorizeAlarm(code) — buckets a raw alarm code into one of:
// - 'overlay' SD-WAN VPN tunnels between peer elements. These
// DO NOT correspond to the physical WAN interfaces
// shown in the Link State / LQM checks; a store
// can have all physical circuits green while the
// overlay flaps repeatedly.
// - 'physical' A specific WAN circuit / interface is affected.
// Should generally line up with what Link State
// reports.
// - 'device' Element / ION appliance itself (heartbeat, boot).
// - 'other' Everything else (LAN, DHCP, config, etc.).
//
// 2. humanizeAlarmCode(code) — one-line English label so the check
// message and the recent-samples list don't dump
// raw Prisma constants at the operator.
//
// 3. rollupAlarms(samples) — groups near-identical alarm samples
// by (code + severity) so 20 successive
// NETWORK_ANYNETLINK_DOWN flaps show as a single
// line with "×20" instead of a 20-line wall.
// Sorted critical → major → minor, then by count.
//
// The category / humanizer tables err on the side of "unknown code
// falls through to a sensible default" — Prisma introduces new codes
// with tenant updates and we do NOT want a new code to blow up the
// diagnostic.
/**
* Alarm category → operator meaning.
*
* overlay → SD-WAN VPN link between peer elements; will NOT
* appear as a physical link-down in the Link State
* check. Common when a hub site or a peer is temporarily
* unreachable.
* physical → An actual WAN circuit / interface is down or
* degraded. Should correlate with Link State / LQM.
* device → Element itself (ION appliance) missed heartbeats,
* rebooted, or entered a bad state.
* other → LAN / DHCP / configuration / anything else.
*/
const CATEGORY_TABLE = {
// ── overlay (VPN tunnel / SD-WAN mesh) ──────────────────────────
NETWORK_ANYNETLINK_DOWN: 'overlay',
NETWORK_VPNLINK_DOWN: 'overlay',
NETWORK_VPNLINK_FLAP: 'overlay',
NETWORK_VPNLINK_UNREACHABLE: 'overlay',
NETWORK_DIRECTPRIVATE_DOWN: 'overlay',
NETWORK_STANDBY_LINK_DOWN: 'overlay',
NETWORK_SITE_UNREACHABLE: 'overlay',
// ── physical (WAN circuit / interface) ──────────────────────────
NETWORK_INTERNET_DOWN: 'physical',
NETWORK_PUBLICWAN_UNREACHABLE: 'physical',
NETWORK_PRIVATEWAN_UNREACHABLE: 'physical',
NETWORK_DIRECTINTERNET_UNREACHABLE: 'physical',
DEVICE_INTERFACE_STATE_CHANGED: 'physical',
DEVICE_INTERFACE_DOWN: 'physical',
DEVICE_INTERFACE_FLAP: 'physical',
NETWORK_LTE_LINK_DOWN: 'physical',
// ── device (element / ION itself) ───────────────────────────────
DEVICE_HB_MISSED: 'device',
DEVICE_UNREACHABLE: 'device',
DEVICE_REBOOT: 'device',
DEVICE_HIGH_CPU: 'device',
DEVICE_HIGH_MEMORY: 'device',
DEVICE_DISK_UTILIZATION_HIGH: 'device',
DEVICE_SOFTWARE_UPGRADE_FAILED: 'device',
DEVICE_HA_STATE_CHANGE: 'device',
// ── other (LAN / DHCP / config) ─────────────────────────────────
DHCP_POOL_UTILIZATION_HIGH: 'other',
DHCP_FAILURE: 'other',
CONFIG_SYNC_FAILED: 'other',
LAN_INTERFACE_DOWN: 'other',
BGP_PEER_DOWN: 'other',
};
/**
* Human-readable one-liner per alarm code. Kept short — operators
* scan the message field, they don't need a paragraph. Falls back to
* a Title-Cased version of the raw code when we don't know it.
*/
const HUMANIZE_TABLE = {
NETWORK_ANYNETLINK_DOWN: 'SD-WAN overlay tunnel down',
NETWORK_VPNLINK_DOWN: 'VPN link to peer site down',
NETWORK_VPNLINK_FLAP: 'VPN link flapping',
NETWORK_VPNLINK_UNREACHABLE: 'VPN link unreachable',
NETWORK_DIRECTPRIVATE_DOWN: 'Direct private link down',
NETWORK_STANDBY_LINK_DOWN: 'Standby link down',
NETWORK_SITE_UNREACHABLE: 'Peer site unreachable',
NETWORK_INTERNET_DOWN: 'Internet WAN circuit down',
NETWORK_PUBLICWAN_UNREACHABLE: 'Public WAN unreachable',
NETWORK_PRIVATEWAN_UNREACHABLE: 'Private WAN unreachable',
NETWORK_DIRECTINTERNET_UNREACHABLE: 'Direct internet unreachable',
NETWORK_LTE_LINK_DOWN: 'LTE link down',
DEVICE_INTERFACE_STATE_CHANGED: 'Physical interface state change',
DEVICE_INTERFACE_DOWN: 'Physical interface down',
DEVICE_INTERFACE_FLAP: 'Physical interface flap',
DEVICE_HB_MISSED: 'ION heartbeat missed',
DEVICE_UNREACHABLE: 'ION unreachable',
DEVICE_REBOOT: 'ION reboot',
DEVICE_HIGH_CPU: 'ION high CPU',
DEVICE_HIGH_MEMORY: 'ION high memory',
DEVICE_DISK_UTILIZATION_HIGH: 'ION disk full',
DEVICE_SOFTWARE_UPGRADE_FAILED: 'Software upgrade failed',
DEVICE_HA_STATE_CHANGE: 'HA state change',
DHCP_POOL_UTILIZATION_HIGH: 'DHCP pool nearly full',
DHCP_FAILURE: 'DHCP failure',
CONFIG_SYNC_FAILED: 'Configuration sync failed',
LAN_INTERFACE_DOWN: 'LAN interface down',
BGP_PEER_DOWN: 'BGP peer down',
};
/**
* Return one of 'overlay' | 'physical' | 'device' | 'other' for the
* given raw alarm code. Case-insensitive on the code. Unknown codes
* bucket to 'other' — Prisma adds new codes over time and we prefer
* a benign fallback to a hard failure.
*/
export function categorizeAlarm(code) {
if (!code) return 'other';
return CATEGORY_TABLE[String(code).toUpperCase()] || 'other';
}
/**
* Return a short human label for the alarm code. Falls back to a
* Title-Cased split of the raw constant when we don't recognise it,
* so `SOMETHING_NEW_DOWN` becomes "Something New Down" instead of
* being dumped raw.
*/
export function humanizeAlarmCode(code) {
if (!code) return 'Unknown alarm';
const key = String(code).toUpperCase();
if (HUMANIZE_TABLE[key]) return HUMANIZE_TABLE[key];
return key
.toLowerCase()
.split('_')
.filter(Boolean)
.map((w) => w.charAt(0).toUpperCase() + w.slice(1))
.join(' ');
}
/**
* Roll alarm samples up by (code + severity). Preserves the newest
* timestamp per rollup and orders results critical → major → minor,
* then by count descending. This turns a wall of 20 near-identical
* NETWORK_ANYNETLINK_DOWN JSON blobs into one scannable line.
*
* Shape of the returned rollups:
* { code, severity, count, newestTs }
*
* (Kept minimal — the caller decorates with humanized label + category
* so we don't force every downstream to import both tables.)
*/
export function rollupAlarms(samples) {
if (!Array.isArray(samples)) return [];
const byKey = new Map();
for (const s of samples) {
if (!s) continue;
const key = `${s.severity}::${s.code}`;
const prev = byKey.get(key);
if (prev) {
prev.count += 1;
if (!prev.newestTs || (s.ts && String(s.ts) > String(prev.newestTs))) {
prev.newestTs = s.ts;
}
} else {
byKey.set(key, {
code: s.code,
severity: s.severity,
count: 1,
newestTs: s.ts,
});
}
}
const sevRank = { critical: 0, major: 1, minor: 2, unknown: 3 };
return [...byKey.values()].sort((a, b) => {
const ra = sevRank[a.severity] ?? 9;
const rb = sevRank[b.severity] ?? 9;
if (ra !== rb) return ra - rb;
return b.count - a.count;
});
}
/**
* Count alarms by category from a set of samples.
* Returns `{ overlay, physical, device, other, total }`.
*/
export function countByCategory(samples) {
const counts = { overlay: 0, physical: 0, device: 0, other: 0, total: 0 };
if (!Array.isArray(samples)) return counts;
for (const s of samples) {
if (!s) continue;
counts.total += 1;
counts[categorizeAlarm(s.code)] += 1;
}
return counts;
}
/**
* Given an alarm timestamp (ISO string), return a coarse "how long
* ago" label. Handles ms drift on both sides — "now", "5m ago",
* "2h ago", "3d ago". Falls back to the raw ts on parse failure.
*/
export function humanizeAge(ts, now = Date.now()) {
if (!ts) return '';
const t = Date.parse(ts);
if (!Number.isFinite(t)) return '';
const deltaSec = Math.max(0, Math.floor((now - t) / 1000));
if (deltaSec < 60) return 'just now';
if (deltaSec < 3600) return `${Math.floor(deltaSec / 60)}m ago`;
if (deltaSec < 86_400) return `${Math.floor(deltaSec / 3600)}h ago`;
return `${Math.floor(deltaSec / 86_400)}d ago`;
}
/**
* Human-friendly minutes label (mirrors the renderer helpers).
* Exported so the alarm check can label windows consistently without
* a separate helper import.
*/
export function humanizeWindow(minutes) {
if (!Number.isFinite(minutes) || minutes <= 0) return `${minutes}m`;
if (minutes >= 1440 && minutes % 1440 === 0) return `${minutes / 1440}d`;
if (minutes >= 60 && minutes % 60 === 0) return `${minutes / 60}h`;
return `${minutes}m`;
}