From 26ae704dff76b271335b9e23ae5d52eb9853d8c1 Mon Sep 17 00:00:00 2001 From: jmcqueen Date: Tue, 28 Jul 2026 09:16:03 -0400 Subject: [PATCH] Tighten WAN tunnel and alarm rendering for actionable follow-ups. Show overall tunnel status with downs only, list the last five alarms with times, and note when overlay/physical alarms may explain bad voice DPI. Co-authored-by: Cursor --- integrations/paloalto/topology.js | 15 ++- services/enrichment/alarmSemantics.js | 37 +++++- services/enrichment/sdwanEnrichment.js | 11 +- services/renderers/wanDiagnosticsRenderer.js | 126 ++++++++++++++----- tests/alarmSemantics.test.js | 16 +++ tests/renderers.wan.test.js | 57 ++++++++- tests/sdwanEnrichment.test.js | 16 ++- 7 files changed, 230 insertions(+), 48 deletions(-) diff --git a/integrations/paloalto/topology.js b/integrations/paloalto/topology.js index 321f046..d4c70fe 100644 --- a/integrations/paloalto/topology.js +++ b/integrations/paloalto/topology.js @@ -17,9 +17,9 @@ import { paloAltoAxios } from './client.js'; import { logger } from '../../utils/logger.js'; -/** Max overlay edges kept per site after ranking. */ -const MAX_TUNNELS = 12; -/** Max status GETs when topology status is missing. */ +/** Soft cap on overlay edges returned (renderer only lists downs). */ +const MAX_TUNNELS = 64; +/** Max status GETs when topology status is missing — prefer downs first. */ const MAX_STATUS_FETCH = 8; /** Topology link types we surface as "overlay tunnels". */ @@ -346,15 +346,18 @@ export async function getVpnLinksForSite(siteId, opts = {}) { return []; } - // Rank: down / unknown first, then cap — never dump dozens of hub paths. + // Prefer down/unknown when capping; keep enough for an accurate + // up/down summary (renderer only lists downs in chat). items = rankAndCapTunnels(items, MAX_TUNNELS); const needsStatus = fetchStatus && items.some((t) => t.up == null); if (needsStatus) { + // Status GETs are expensive — hydrate downs/unknowns first. const statusIds = []; - for (const t of items) { - if (t.up != null) continue; + const rankedForStatus = [...items].sort((a, b) => scoreTunnel(b) - scoreTunnel(a)); + for (const t of rankedForStatus) { + if (t.up === true) continue; if (t.vpnLinkIds?.length) statusIds.push(t.vpnLinkIds[0]); else if (t.id) statusIds.push(t.id); if (statusIds.length >= MAX_STATUS_FETCH) break; diff --git a/services/enrichment/alarmSemantics.js b/services/enrichment/alarmSemantics.js index c6d2f78..815feb1 100644 --- a/services/enrichment/alarmSemantics.js +++ b/services/enrichment/alarmSemantics.js @@ -54,6 +54,8 @@ const CATEGORY_TABLE = { NETWORK_DIRECTPRIVATE_DOWN: 'overlay', NETWORK_STANDBY_LINK_DOWN: 'overlay', NETWORK_SITE_UNREACHABLE: 'overlay', + SITE_CONNECTIVITY_DEGRADED: 'overlay', + NETWORK_SITE_CONNECTIVITY_DEGRADED: 'overlay', // ── physical (WAN circuit / interface) ────────────────────────── NETWORK_INTERNET_DOWN: 'physical', @@ -96,6 +98,8 @@ const HUMANIZE_TABLE = { NETWORK_DIRECTPRIVATE_DOWN: 'Direct private link down', NETWORK_STANDBY_LINK_DOWN: 'Standby link down', NETWORK_SITE_UNREACHABLE: 'Peer site unreachable', + SITE_CONNECTIVITY_DEGRADED: 'Site connectivity degraded', + NETWORK_SITE_CONNECTIVITY_DEGRADED: 'Site connectivity degraded', NETWORK_INTERNET_DOWN: 'Internet WAN circuit down', NETWORK_PUBLICWAN_UNREACHABLE: 'Public WAN unreachable', NETWORK_PRIVATEWAN_UNREACHABLE: 'Private WAN unreachable', @@ -190,6 +194,15 @@ export function rollupAlarms(samples) { }); } +/** + * Overlay / physical / device alarms are the ones most likely to + * explain poor voice quality. LAN/DHCP/config usually do not. + */ +export function isVoiceRelevantAlarm(code) { + const cat = categorizeAlarm(code); + return cat === 'overlay' || cat === 'physical' || cat === 'device'; +} + /** * Count alarms by category from a set of samples. * Returns `{ overlay, physical, device, other, total }`. @@ -205,6 +218,24 @@ export function countByCategory(samples) { return counts; } +/** + * Coerce Prisma alarm time (ISO string or epoch ms) to ms since epoch. + * CloudGenix sometimes stores µs-scale ids (>1e14) — those are scaled down. + */ +export function coerceAlarmTsMs(ts) { + if (ts == null || ts === '') return null; + if (typeof ts === 'number' && Number.isFinite(ts)) { + return ts > 1e14 ? Math.floor(ts / 1000) : ts; + } + const asStr = String(ts).trim(); + if (/^\d+$/.test(asStr)) { + const n = Number(asStr); + return n > 1e14 ? Math.floor(n / 1000) : n; + } + const parsed = Date.parse(asStr); + return Number.isFinite(parsed) ? parsed : null; +} + /** * Given an alarm timestamp (ISO string), return a coarse "how long * ago" label. Handles ms drift on both sides — "now", "5m ago", @@ -212,9 +243,9 @@ export function countByCategory(samples) { */ export function humanizeAge(ts, now = Date.now()) { if (!ts) return ''; - const t = Date.parse(ts); - if (!Number.isFinite(t)) return ''; - const deltaSec = Math.max(0, Math.floor((now - t) / 1000)); + const ms = coerceAlarmTsMs(ts); + if (ms == null) return ''; + const deltaSec = Math.max(0, Math.floor((now - ms) / 1000)); if (deltaSec < 60) return 'just now'; if (deltaSec < 3600) return `${Math.floor(deltaSec / 60)}m ago`; if (deltaSec < 86_400) return `${Math.floor(deltaSec / 3600)}h ago`; diff --git a/services/enrichment/sdwanEnrichment.js b/services/enrichment/sdwanEnrichment.js index 86d2d8d..9afb11c 100644 --- a/services/enrichment/sdwanEnrichment.js +++ b/services/enrichment/sdwanEnrichment.js @@ -77,6 +77,7 @@ import { } from '../../integrations/paloalto/metrics.js'; import { getVpnLinksForSite, extractVpnLinkIdsFromAlarmInfo } from '../../integrations/paloalto/topology.js'; import { buildAppDetailsUrl } from '../../integrations/paloalto/urls.js'; +import { coerceAlarmTsMs } from './alarmSemantics.js'; /** * Compose everything /voicediag + /phonestatus care about for a @@ -940,10 +941,16 @@ export function parseAlarms(resp, siteId = null) { vpnLinkIds, }); } - samples.sort((a, b) => String(b.ts || '').localeCompare(String(a.ts || ''))); + samples.sort((a, b) => { + const ta = coerceAlarmTsMs(a.ts) ?? 0; + const tb = coerceAlarmTsMs(b.ts) ?? 0; + return tb - ta; + }); return { last1h: counts, - samples: samples.slice(0, 5), + // Keep enough samples for accurate code rollups; the renderer + // shows the newest 5 individually with timestamps. + samples: samples.slice(0, 50), // Diagnostic surface for the caller — lets sdwanEnrichment log // "raw N events → K after site filter" so we can see whether // Prisma's server-side site filter is working or being ignored. diff --git a/services/renderers/wanDiagnosticsRenderer.js b/services/renderers/wanDiagnosticsRenderer.js index fa007ea..500792b 100644 --- a/services/renderers/wanDiagnosticsRenderer.js +++ b/services/renderers/wanDiagnosticsRenderer.js @@ -21,6 +21,8 @@ import { humanizeAlarmCode, rollupAlarms as sharedRollupAlarms, humanizeAge, + isVoiceRelevantAlarm, + coerceAlarmTsMs, } from '../enrichment/alarmSemantics.js'; import { formatDisplayTime } from '../../utils/time.js'; @@ -93,7 +95,7 @@ export function renderWanDiagnosticsMarkdown(data, opts = {}) { out += `_No WAN path metrics available for this site._\n`; } - // Overlay / VPN tunnels (Phase 1). + // Overlay / VPN tunnels — overall status; list downs only. out += renderTunnelsSection(data.tunnels, data.links); // Per-app "Application Path Details" section — real voice-quality @@ -110,11 +112,9 @@ export function renderWanDiagnosticsMarkdown(data, opts = {}) { out += renderAppAudioSection(data.appAudio, data.window); } - // Alarms summary (only if any are active). Samples are rolled up - // by (code + severity) so 20 identical NETWORK_ANYNETLINK_DOWN - // events show as one line with "×20" instead of pasting 20 nearly- - // identical JSON blobs into chat. Full alarm detail belongs in the - // Prisma UI — this is a summary surface. + // Alarms: severity totals + code rollup + last 5 with timestamps. + // When voice DPI looks bad and recent alarms are overlay/physical/ + // device, call out the correlation (not proof of causality). const totalAlarms = (data.alarms?.last1h?.critical || 0) + (data.alarms?.last1h?.major || 0) + @@ -130,7 +130,8 @@ export function renderWanDiagnosticsMarkdown(data, opts = {}) { : '1h'; out += `\n🚨 Alarms (last ${alarmWindowLabel}): ${parts.join(', ')}\n`; - const rollups = sharedRollupAlarms(data.alarms.samples || []); + const samples = data.alarms.samples || []; + const rollups = sharedRollupAlarms(samples); for (const r of rollups.slice(0, 5)) { const label = humanizeAlarmCode(r.code); const category = categorizeAlarm(r.code); @@ -143,6 +144,22 @@ export function renderWanDiagnosticsMarkdown(data, opts = {}) { if (rollups.length > 5) { out += ` - _+${rollups.length - 5} more alarm code${rollups.length - 5 === 1 ? '' : 's'}._\n`; } + + const recent = samples.slice(0, 5); + if (recent.length > 0) { + out += ` _Most recent:_\n`; + for (const s of recent) { + const label = humanizeAlarmCode(s.code); + const when = formatAlarmWhen(s.ts); + const voiceBit = isVoiceRelevantAlarm(s.code) ? ' · may affect voice' : ''; + out += + ` - ${sevIcon(s.severity)} **${label}**` + + `${when ? ` — ${when}` : ''}${voiceBit}\n`; + } + } + + const voiceCorr = voiceAlarmCorrelationNote(data.appAudio, samples); + if (voiceCorr) out += ` _${voiceCorr}_\n`; } // Per-metric fetch failures shown as small warnings so the operator @@ -270,47 +287,92 @@ function renderTunnelsSection(tunnels, links) { && links.length > 0 && links.every((l) => l.up !== false); - const actionable = list.filter((t) => t.up === false || t.recentAlarm || t.up === true); - const unknownOnly = list.length > 0 && actionable.every((t) => t.up == null) - && list.every((t) => t.up == null); + const down = list.filter((t) => t.up === false); + const up = list.filter((t) => t.up === true); + const unknown = list.filter((t) => t.up == null); + const total = list.length; - let out = `\n**Overlay tunnels** (${list.length})\n`; + let summary; + if (down.length === 0 && unknown.length === 0) { + summary = `✅ all ${total} up`; + } else if (down.length === 0 && up.length === 0) { + summary = `❓ ${total} inventoried (status unknown)`; + } else if (down.length === 0) { + summary = `✅ ${up.length}/${total} up` + + (unknown.length ? ` · ${unknown.length} unknown` : ''); + } else { + summary = `❌ ${down.length} down / ${total}` + + (up.length ? ` · ${up.length} up` : '') + + (unknown.length ? ` · ${unknown.length} unknown` : ''); + } - // If every tunnel is unparseable unknown, don't spam "? peer — unknown". - if (unknownOnly) { - out += `- ❓ Tunnel inventory returned ${list.length} link(s) but peer/status fields were empty — check Prisma UI topology for this site.\n`; + let out = `\n**Overlay tunnels** — ${summary}\n`; + + // All-unknown: one inventory note, no peer spam. + if (down.length === 0 && up.length === 0 && unknown.length > 0) { + out += `- ❓ Tunnel inventory returned ${total} link(s) but peer/status fields were empty — check Prisma UI topology for this site.\n`; return out; } - const ranked = [...list].sort((a, b) => { - const score = (t) => (t.up === false ? 3 : (t.recentAlarm ? 2 : (t.up === true ? 1 : 0))); - return score(b) - score(a); - }); - // Prefer showing down / alarmed first; skip pure-unknown fillers when - // we already have actionable rows. Cap hard so we never dump dozens. - const toShow = ranked.filter((t) => t.up !== null || t.recentAlarm || t.peerLabel !== 'peer'); - const display = (toShow.length > 0 ? toShow : ranked).slice(0, 6); - for (const t of display) { - const icon = t.up === null ? '❓' : (t.up ? '✅' : '❌'); - const state = t.up === null ? (t.state || 'unknown') : (t.up ? 'up' : 'DOWN'); + // Only list downs. Healthy tunnels stay in the summary line. + const toShow = [...down].sort((a, b) => { + if (a.recentAlarm !== b.recentAlarm) return a.recentAlarm ? -1 : 1; + return String(a.peerLabel || '').localeCompare(String(b.peerLabel || '')); + }).slice(0, 8); + + for (const t of toShow) { const peer = (t.peerLabel && t.peerLabel !== 'peer') ? t.peerLabel : (t.peerSiteId || 'peer'); - const ifBit = t.relatedInterfaceId ? ` · if ${t.relatedInterfaceId}` : ''; const circuitBit = t.circuitName ? ` · ${t.circuitName}` : ''; + const ifBit = t.relatedInterfaceId ? ` · if ${t.relatedInterfaceId}` : ''; const alarmBit = t.recentAlarm ? ' · ⚠️ recent alarm' : ''; - out += `- ${icon} **${peer}** — ${state}${circuitBit}${ifBit}${alarmBit}\n`; + out += `- ❌ **${peer}** — DOWN${circuitBit}${ifBit}${alarmBit}\n`; } - if (ranked.length > display.length) { - out += `- _+${ranked.length - display.length} more tunnel(s)._\n`; + if (down.length > toShow.length) { + out += `- _+${down.length - toShow.length} more down tunnel(s)._\n`; } - const downCount = list.filter((t) => t.up === false).length; - if (downCount > 0 && physicalUp) { - out += `_Note: ${downCount} overlay tunnel(s) down while physical WAN paths look up — voice may still be impacted._\n`; + if (down.length > 0 && physicalUp) { + out += `_Note: overlay down while physical WAN paths look up — voice may still be impacted._\n`; } return out; } +/** + * Format an alarm timestamp for chat: wall clock + relative age. + */ +function formatAlarmWhen(ts) { + const ms = coerceAlarmTsMs(ts); + if (ms == null) return humanizeAge(ts) || ''; + const wall = formatDisplayTime(new Date(ms)); + const age = humanizeAge(new Date(ms).toISOString()); + return age ? `${wall} (${age})` : wall; +} + +/** + * Heuristic: when voice DPI is degraded and recent alarms are + * overlay/physical/device, note possible correlation (not causation). + */ +function voiceAlarmCorrelationNote(appAudio, samples) { + if (!appAudio || !Array.isArray(samples) || samples.length === 0) return ''; + if (!voiceQualityLooksDegraded(appAudio)) return ''; + const relevant = samples.filter((s) => isVoiceRelevantAlarm(s.code)); + if (relevant.length === 0) return ''; + const cats = new Set(relevant.map((s) => categorizeAlarm(s.code))); + const catList = [...cats].join('/'); + return ( + `Voice DPI looks degraded in this window and ${relevant.length} ` + + `${catList} alarm(s) may help explain it (correlation, not proof).` + ); +} + +function voiceQualityLooksDegraded(appAudio) { + const mosBad = appAudio.mos?.validSamples > 0 && appAudio.mos.min != null && appAudio.mos.min < 3.5; + const lossBad = appAudio.loss?.validSamples > 0 && appAudio.loss.max != null && appAudio.loss.max > 5; + const jitterBad = appAudio.jitter?.validSamples > 0 && appAudio.jitter.max != null && appAudio.jitter.max > 50; + return mosBad || lossBad || jitterBad; +} + /** * One row of the voice-traffic-quality section. * diff --git a/tests/alarmSemantics.test.js b/tests/alarmSemantics.test.js index 72cb096..b59628e 100644 --- a/tests/alarmSemantics.test.js +++ b/tests/alarmSemantics.test.js @@ -21,6 +21,8 @@ import { countByCategory, humanizeAge, humanizeWindow, + isVoiceRelevantAlarm, + coerceAlarmTsMs, } from '../services/enrichment/alarmSemantics.js'; // ─── categorizeAlarm ───────────────────────────────────────────────── @@ -32,6 +34,7 @@ test('categorizeAlarm: overlay codes bucket as overlay', () => { 'NETWORK_VPNLINK_FLAP', 'NETWORK_SITE_UNREACHABLE', 'NETWORK_STANDBY_LINK_DOWN', + 'SITE_CONNECTIVITY_DEGRADED', ]) { assert.equal(categorizeAlarm(code), 'overlay', `${code} → overlay`); } @@ -179,3 +182,16 @@ test('humanizeWindow: canonical values', () => { assert.equal(humanizeWindow(10080), '7d', 'the new default WAN window must round-trip cleanly'); }); + +test('isVoiceRelevantAlarm: overlay/physical/device yes, other no', () => { + assert.equal(isVoiceRelevantAlarm('NETWORK_VPNLINK_DOWN'), true); + assert.equal(isVoiceRelevantAlarm('NETWORK_INTERNET_DOWN'), true); + assert.equal(isVoiceRelevantAlarm('DEVICE_REBOOT'), true); + assert.equal(isVoiceRelevantAlarm('DHCP_FAILURE'), false); +}); + +test('coerceAlarmTsMs: ISO and epoch', () => { + assert.equal(coerceAlarmTsMs('2025-01-01T00:00:00Z'), Date.parse('2025-01-01T00:00:00Z')); + assert.equal(coerceAlarmTsMs(1_700_000_000_000), 1_700_000_000_000); + assert.equal(coerceAlarmTsMs(null), null); +}); diff --git a/tests/renderers.wan.test.js b/tests/renderers.wan.test.js index 21cc965..20f79f6 100644 --- a/tests/renderers.wan.test.js +++ b/tests/renderers.wan.test.js @@ -280,16 +280,31 @@ test('renderer: footer mentions the per-app --only shortcut', () => { 'footer should point operators at the per-app checks by id'); }); -test('renderer: overlay tunnels section shows down + physical-up note', () => { +test('renderer: overlay tunnels show overall status and only list downs', () => { const md = renderWanDiagnosticsMarkdown(baseData({ tunnels: [ { id: 't1', peerLabel: 'CG00001-HUB', up: false, state: 'down', recentAlarm: true }, { id: 't2', peerLabel: 'CG00002-HUB', up: true, state: 'up' }, + { id: 't3', peerLabel: 'Cologix', up: true, state: 'up' }, ], }), { storeNum: '782', footer: false }); - assert.match(md, /\*\*Overlay tunnels\*\*/); + assert.match(md, /\*\*Overlay tunnels\*\* — ❌ 1 down \/ 3/); assert.match(md, /CG00001-HUB/); - assert.match(md, /overlay tunnel\(s\) down while physical/); + assert.doesNotMatch(md, /CG00002-HUB/); + assert.doesNotMatch(md, /Cologix/); + assert.match(md, /overlay down while physical/); +}); + +test('renderer: all-up tunnels collapse to summary only', () => { + const md = renderWanDiagnosticsMarkdown(baseData({ + tunnels: [ + { id: 't1', peerLabel: 'Warrendale', up: true, state: 'up' }, + { id: 't2', peerLabel: 'Cologix', up: true, state: 'up' }, + ], + }), { storeNum: '782', footer: false }); + assert.match(md, /\*\*Overlay tunnels\*\* — ✅ all 2 up/); + assert.doesNotMatch(md, /Warrendale/); + assert.doesNotMatch(md, /Cologix/); }); test('renderer: all-unknown tunnels collapse to one inventory note', () => { @@ -298,10 +313,46 @@ test('renderer: all-unknown tunnels collapse to one inventory note', () => { id: `t${i}`, peerLabel: 'peer', up: null, state: 'unknown', })), }), { storeNum: '782', footer: false }); + assert.match(md, /status unknown/); assert.match(md, /Tunnel inventory returned 5/); assert.doesNotMatch(md, /\? peer — unknown/); }); +test('renderer: alarms include most-recent list with times and voice correlation', () => { + const now = Date.now(); + const md = renderWanDiagnosticsMarkdown(baseData({ + appAudio: { + appName: 'Webex_Calling_RTP', + mos: { avg: 4.0, min: 1.0, max: 4.4, samples: 10, validSamples: 10, interval: '5min' }, + loss: { avg: 4, min: 0, max: 68, samples: 10, validSamples: 10, interval: '5min' }, + jitter: { avg: 1, min: 0, max: 2, samples: 10, validSamples: 10, interval: '5min' }, + bandwidth: { avg: 0.5, min: 0.1, max: 1, samples: 10, validSamples: 10, interval: '5min' }, + }, + alarms: { + last1h: { critical: 0, major: 2, minor: 0 }, + samples: [ + { + code: 'SITE_CONNECTIVITY_DEGRADED', + severity: 'major', + message: 'degraded', + ts: new Date(now - 9 * 3600 * 1000).toISOString(), + }, + { + code: 'NETWORK_VPNLINK_DOWN', + severity: 'major', + message: 'down', + ts: new Date(now - 10 * 3600 * 1000).toISOString(), + }, + ], + }, + }), { storeNum: '782', footer: false }); + assert.match(md, /Most recent:/); + assert.match(md, /Site connectivity degraded/i); + assert.match(md, /may affect voice/); + assert.match(md, /Voice DPI looks degraded/); + assert.match(md, /may help explain it/); +}); + test('renderer: voice DPI byPathType lines', () => { const md = renderWanDiagnosticsMarkdown(baseData({ appAudio: { diff --git a/tests/sdwanEnrichment.test.js b/tests/sdwanEnrichment.test.js index 07068e3..0e016dc 100644 --- a/tests/sdwanEnrichment.test.js +++ b/tests/sdwanEnrichment.test.js @@ -510,7 +510,7 @@ test('parseAlarms: nil → empty counts', () => { assert.deepEqual(a.samples, []); }); -test('parseAlarms: counts by severity, keeps 5 most-recent samples', () => { +test('parseAlarms: counts by severity, keeps up to 50 samples (newest first)', () => { const resp = { items: [ { severity: 'critical', code: 'C1', info: 'boom', time: '2025-01-01T10:00:00Z' }, @@ -525,10 +525,22 @@ test('parseAlarms: counts by severity, keeps 5 most-recent samples', () => { assert.equal(a.last1h.critical, 1); assert.equal(a.last1h.major, 2); assert.equal(a.last1h.minor, 3); - assert.equal(a.samples.length, 5); + assert.equal(a.samples.length, 6); assert.equal(a.samples[0].code, 'C1', 'newest first'); }); +test('parseAlarms: caps samples at 50', () => { + const items = Array.from({ length: 60 }, (_, i) => ({ + severity: 'minor', + code: `C${i}`, + info: 'x', + time: new Date(Date.UTC(2025, 0, 1, 12, 0, i)).toISOString(), + })); + const a = parseAlarms({ items }); + assert.equal(a.samples.length, 50); + assert.equal(a.last1h.minor, 60); +}); + test('parseAlarms: filters out cleared alarms (events/query returns both open + cleared)', () => { const resp = { items: [