diff --git a/.env.example b/.env.example index 6d403ae..37f5120 100644 --- a/.env.example +++ b/.env.example @@ -81,6 +81,78 @@ WEBEX_TOKENS_PATH=./config/webex-service-tokens.json # Optional override for the Webex API base URL (default https://webexapis.com/v1). # WEBEX_BASE_URL=https://webexapis.com/v1 +# ----------------------------------------------------------------------------- +# Palo Alto Prisma SD-WAN (formerly CloudGenix) — WAN metrics for /voicediag +# and /phonestatus follow-up +# ----------------------------------------------------------------------------- +# The Prisma integration is optional: leave it unconfigured and the WAN +# check bucket in /voicediag skips cleanly ("not applicable — missing +# sdwanSite") and /phonestatus omits the WAN follow-up message. Enable it +# by setting PRISMA_AUTH_MODE + the matching credential block below. +# +# AE convention (encoded in integrations/paloalto/sites.js): Prisma site +# names are `CG${storeNum.padStart(5, '0')}` — store 782 → CG00782, store +# 2477 → CG02477. Exact match. No fuzzy searching. +# +# --- Auth mode selection --- +# PRISMA_AUTH_MODE=sase (default, recommended for production) +# PRISMA_AUTH_MODE=legacy (only if you inherited a CloudGenix service +# account and can't provision a SASE one) +# PRISMA_AUTH_MODE= + +# --- Unified SASE OAuth 2.0 (PRISMA_AUTH_MODE=sase) --- +# Provision a service-app credential in Strata Cloud Manager → Identity & +# Access → Service Accounts. TSG_ID is the Tenant Service Group id shown +# in the same panel — it scopes the resulting bearer token to your tenant. +# Both defaults below are correct for the public SASE cloud; override for +# region-specific gateways if you're on one. +# PRISMA_CLIENT_ID= +# PRISMA_CLIENT_SECRET= +# PRISMA_TSG_ID= +# PRISMA_SASE_BASE_URL=https://api.sase.paloaltonetworks.com +# PRISMA_AUTH_URL=https://auth.apps.paloaltonetworks.com/oauth2/access_token + +# --- Legacy CloudGenix session token (PRISMA_AUTH_MODE=legacy) --- +# Interactive-user credential (email + password). Kept for backward +# compatibility only — the SASE mode above is what new deployments should +# use because it's service-account-shaped instead of impersonating a +# human. Session tokens last ~8 hours; the client transparently +# re-authenticates on 401. +# PRISMA_EMAIL= +# PRISMA_PASSWORD= +# PRISMA_LEGACY_BASE_URL=https://api.cloudgenix.com + +# --- WAN voice-quality thresholds (used by /voicediag WAN bucket + the +# /phonestatus WAN follow-up renderer) --- +# Defaults follow ITU-T G.114 (latency), G.711 PLC (loss), and RFC 3550 +# (jitter) references. See services/voiceDiag/README.md for the standards +# reference table. +# WAN_STANDARD_LATENCY_WARN_MS=150 +# WAN_STANDARD_LATENCY_ERROR_MS=400 +# WAN_STANDARD_JITTER_WARN_MS=30 +# WAN_STANDARD_JITTER_ERROR_MS=50 +# WAN_STANDARD_LOSS_WARN_PCT=1 +# WAN_STANDARD_LOSS_ERROR_PCT=3 +# WAN_STANDARD_MOS_WARN=4.0 +# WAN_STANDARD_MOS_ERROR=3.5 +# WAN_STANDARD_HEALTHSCORE_WARN=80 +# WAN_STANDARD_HEALTHSCORE_ERROR=60 + +# --- WAN bucket global kill-switch (mirror of VOICE_STANDARD_ENABLED for +# the port bucket). Set to `false` to silence the entire WAN check +# bucket while a Prisma cleanup is in progress. The /phonestatus +# follow-up is unaffected — it always renders whatever Prisma +# returns. Feature-config + port-hygiene buckets keep running. --- +# WAN_STANDARD_ENABLED=true + +# --- WAN look-back window (minutes) used when the operator doesn't +# pass `--window` on /voicediag. Applies to healthscore + LQM. +# Alarms window is max(60, WAN_STANDARD_WINDOW_MINUTES) since +# sub-hour alarm queries are usually too noisy to be actionable. +# Prisma's finest LQM bucket is 5m; anything <= 5 is snapped up. +# Common values: 15 (default, real-time), 60, 360 (6h), 1440 (24h). +# WAN_STANDARD_WINDOW_MINUTES=15 + # ----------------------------------------------------------------------------- # /voicediag — Store voice-line standards # ----------------------------------------------------------------------------- diff --git a/.gitignore b/.gitignore index c21de4e..9534de0 100644 --- a/.gitignore +++ b/.gitignore @@ -31,6 +31,7 @@ scripts/* !scripts/reclaimWebexHosts.js !scripts/removeAdvancedMessaging.js !scripts/findEmptyLocations.js +!scripts/prismaProbe.js !scripts/lib/ !scripts/lib/** characterize-*.js diff --git a/commands/phoneStatus.js b/commands/phoneStatus.js index 8328cee..f4b5986 100644 --- a/commands/phoneStatus.js +++ b/commands/phoneStatus.js @@ -12,12 +12,15 @@ import { renderPhoneStatusMarkdown, renderDectDiagnosticsMarkdown, } from '../services/renderers/phoneStatusRenderer.js'; +import { renderWanDiagnosticsMarkdown } from '../services/renderers/wanDiagnosticsRenderer.js'; import { buildIgmpFixCard } from './igmpFix.js'; import { pendingIgmpFixes } from '../utils/pendingIgmpFixes.js'; import { extractRequester } from '../utils/requester.js'; import { logger } from '../utils/logger.js'; import { discoverDectBases } from '../services/dectDiscovery.js'; import { collectAll } from '../services/dectCollectorService.js'; +import { siteNameForStore, findSdwanSiteForStore } from '../integrations/paloalto/sites.js'; +import { collectSdwanForStore } from '../services/enrichment/sdwanEnrichment.js'; export async function handlePhoneStatus(bot, trigger) { logger('phone:status', 'Handler entered', 'debug'); @@ -77,11 +80,43 @@ export async function handlePhoneStatus(bot, trigger) { ); } + // WAN follow-up discovery. Cheap Prisma-side check: hits the + // 4-hour-cached sites list to confirm this store is Prisma- + // managed before we promise a follow-up. Anything that throws + // (missing env, auth failure, network error) is swallowed and + // treated as "no site" so a broken Prisma integration cannot + // break /phonestatus. Chat-only, same as DECT. + let wanFollowUpEnabled = false; + if (trigger.person) { + const expectedSite = safeSiteName(storeNum); + try { + const site = await findSdwanSiteForStore(storeNum); + wanFollowUpEnabled = !!site; + // Info-level breadcrumb so operators can see the discovery + // outcome without cranking LOG_LEVEL=debug. The success case + // is also logged inside findSdwanSiteForStore; this line + // provides the "why /phonestatus did / didn't schedule a + // WAN follow-up" answer at the phone:status scope. + logger( + 'phone:status', + `WAN discovery for store ${storeNum} (expected ${expectedSite}): ` + + `${wanFollowUpEnabled ? 'MATCHED — follow-up scheduled' : 'no match — no follow-up'}`, + ); + } catch (err) { + logger( + 'phone:status', + `WAN discovery skipped for store ${storeNum} (expected ${expectedSite}): ${err.message}`, + 'warn', + ); + } + } + const reply = renderPhoneStatusMarkdown(data, { storeNum, detailed: isDetailed, footer: true, dectFollowUpBaseCount: reachableBases.length, + wanFollowUpEnabled, }); await bot.say('markdown', reply || 'No data available.'); @@ -98,6 +133,15 @@ export async function handlePhoneStatus(bot, trigger) { }); } + // WAN follow-up (mirrors the DECT pattern). Only kicked when + // discovery above already confirmed we have a Prisma site for + // this store. Fire-and-forget with a stable log scope. + if (wanFollowUpEnabled) { + runWanFollowUp(bot, storeNum).catch((err) => { + logger('phone:status', `WAN follow-up failed for store ${storeNum}: ${err.message}`, 'error'); + }); + } + // IGMP-snooping remediation card — only when (a) the multicast // summary flagged deviation AND (b) we know the networkId (can't // fix what we can't address) AND (c) the invocation came from @@ -165,3 +209,36 @@ async function runDectFollowUp(bot, storeNum, bases) { if (!md) return; await bot.say('markdown', md); } + +/** + * Prisma SD-WAN follow-up. Runs the enrichment composer (which + * hydrates site + elements + healthscore + per-path LQM + alarms in + * parallel), hands the result to the pure WAN renderer, and posts. + * + * The composer never throws — every metric failure is preserved in + * `data.errors[]` and rendered as an inline "partial fetch" warning + * so the operator can see WHAT failed rather than getting silence. + * A missing site (unexpected here since we pre-discovered) short- + * circuits with an empty markdown string, and we no-op. + */ +async function runWanFollowUp(bot, storeNum) { + const data = await collectSdwanForStore(storeNum); + const md = renderWanDiagnosticsMarkdown(data, { storeNum }); + if (!md) return; + await bot.say('markdown', md); +} + +/** + * Small helper for a log line that runs before we've committed to + * a site lookup — used inside the catch branch of WAN discovery + * where we want to show the expected site name even if the lookup + * failed. Isolated in a function so the try/catch is single-line + * and the intent is obvious. + */ +function safeSiteName(storeNum) { + try { + return siteNameForStore(storeNum); + } catch { + return ''; + } +} diff --git a/commands/voiceDiag.js b/commands/voiceDiag.js index 013ae21..da76e1a 100644 --- a/commands/voiceDiag.js +++ b/commands/voiceDiag.js @@ -75,6 +75,19 @@ export async function handleVoiceDiag(bot, trigger) { .map((s) => s.trim()) .filter(Boolean); + // Look-back window for WAN checks (healthscore + LQM + alarms). + // Accepts `15m`, `1h`, `24h`, `1d` etc. Defaults to whatever + // `buildContext` resolves via env WAN_STANDARD_WINDOW_MINUTES. + const windowRaw = pickWindowArg(args) ?? query.window ?? null; + const windowMinutes = parseWindowMinutes(windowRaw); + if (windowRaw && windowMinutes == null) { + await bot.say( + 'markdown', + `Unrecognised window \`${windowRaw}\`. Use \`15m\`, \`1h\`, \`6h\`, \`24h\`, or \`1d\`.`, + ); + return; + } + if (!storeNum || !/^\d{2,4}$/.test(storeNum)) { await bot.say( 'markdown', @@ -82,6 +95,7 @@ export async function handleVoiceDiag(bot, trigger) { 'Examples:\n' + '- `/voicediag 782`\n' + '- `/voicediag 782 detailed`\n' + + '- `/voicediag 782 --window 24h`\n' + '- `/voicediag 782 --only dnd,callForwarding`\n' + '- `/voicediag list-checks`', ); @@ -90,7 +104,7 @@ export async function handleVoiceDiag(bot, trigger) { let ctx; try { - ctx = await buildContext(storeNum); + ctx = await buildContext(storeNum, { windowMinutes }); } catch (err) { logger('voicediag', `buildContext failed for store ${storeNum}: ${err.message}`, 'error'); await bot.say('markdown', `❌ Failed to build diagnostic context for store ${storeNum}: ${err.message}`); @@ -122,6 +136,7 @@ export async function handleVoiceDiag(bot, trigger) { personLabel: ctx.personLabel, email: ctx.email, detailed, + wanWindowMinutes: ctx.sdwanData?.window?.minutes ?? null, }); await bot.say('markdown', reply || 'No diagnostic results.'); @@ -327,18 +342,45 @@ export function normalizeArg(raw) { // Supports both `--only=dnd,callWaiting` and `--only dnd,callWaiting` // (i.e. the value is the token immediately after `--only`). function pickOnlyArg(args) { + return pickFlagValue(args, '--only'); +} + +function pickWindowArg(args) { + return pickFlagValue(args, '--window'); +} + +// Generic `--flag=value` / `--flag value` extractor. +function pickFlagValue(args, flag) { + const prefix = `${flag.toLowerCase()}=`; for (let i = 0; i < args.length; i += 1) { - const a = String(args[i] || ''); - if (a.toLowerCase().startsWith('--only=')) { - return a.slice('--only='.length); - } - if (a.toLowerCase() === '--only' && i + 1 < args.length) { - return args[i + 1]; - } + const a = String(args[i] || '').toLowerCase(); + if (a.startsWith(prefix)) return String(args[i]).slice(prefix.length); + if (a === flag.toLowerCase() && i + 1 < args.length) return args[i + 1]; } return null; } +/** + * Parse a window shorthand (`15m`, `1h`, `24h`, `1d`, `1440`) into + * minutes. Returns null on unrecognised input so the caller can + * decide whether to surface a friendly error or fall back to the + * env default. + */ +export function parseWindowMinutes(raw) { + if (raw === null || raw === undefined || raw === '') return undefined; + const s = String(raw).trim().toLowerCase(); + // Bare integer → treat as minutes. + if (/^\d+$/.test(s)) return Math.max(1, parseInt(s, 10)); + const m = s.match(/^(\d+)\s*(m|min|mins|h|hr|hrs|hour|hours|d|day|days)$/); + if (!m) return null; + const n = parseInt(m[1], 10); + const unit = m[2]; + if (['m', 'min', 'mins'].includes(unit)) return n; + if (['h', 'hr', 'hrs', 'hour', 'hours'].includes(unit)) return n * 60; + if (['d', 'day', 'days'].includes(unit)) return n * 60 * 24; + return null; +} + function renderListChecks() { let md = '**/voicediag registered checks**\n\n'; for (const c of CHECKS) { diff --git a/integrations/paloalto/client.js b/integrations/paloalto/client.js new file mode 100644 index 0000000..12be373 --- /dev/null +++ b/integrations/paloalto/client.js @@ -0,0 +1,361 @@ +// src/integrations/paloalto/client.js +// +// Palo Alto Prisma SD-WAN (formerly CloudGenix) HTTP client. Wraps +// axios with dual-mode auth so the same client works against the +// unified SASE OAuth 2.0 surface (`api.sase.paloaltonetworks.com`) +// and the legacy CloudGenix session-token surface +// (`api.cloudgenix.com`) without any caller-visible differences. +// +// Auth mode selection lives in `PRISMA_AUTH_MODE`: +// - `sase` (default, recommended for production) — service-app +// OAuth 2.0 client_credentials grant. Bearer token echoed on +// every request. Access tokens are short-lived (~15 min); we +// refresh 5 min ahead of expiry and forcibly refresh on the +// first 401 with a one-shot retry. +// - `legacy` — CloudGenix session-token flow. Interactive-user +// credential (email + password), captured as `x-auth-token` +// for every subsequent call. Kept for admins who only have +// legacy creds provisioned; new deployments should prefer +// `sase`. +// +// The single `paloAltoAxios` export is what business modules +// import — same pattern as `scAxios` / `merakiAxios`. Mutex around +// token refresh (same reason as Webex — a race would double-spend +// the refresh and leave one caller with a stale token). +// +// Only READ operations are used by /voicediag today (POSTs to the +// `/monitor/v2.0/api/monitor/*` metric endpoints). No PUT/DELETE +// surface is invoked from this client, so a config mishap during +// token refresh cannot mutate SD-WAN state. + +import axios from 'axios'; +import { Mutex } from 'async-mutex'; +import { logger } from '../../utils/logger.js'; + +const AUTH_MODE_SASE = 'sase'; +const AUTH_MODE_LEGACY = 'legacy'; + +const DEFAULT_SASE_BASE_URL = 'https://api.sase.paloaltonetworks.com'; +const DEFAULT_SASE_AUTH_URL = 'https://auth.apps.paloaltonetworks.com/oauth2/access_token'; +const DEFAULT_LEGACY_BASE_URL = 'https://api.cloudgenix.com'; + +// Refresh window: 5 minutes before actual token expiry. Prisma +// SASE tokens are 900s by default; a request landing at 895s could +// otherwise trip a 401 that we then re-authenticate through — not +// wrong, just wasteful. +const REFRESH_MARGIN_MS = 5 * 60 * 1000; + +const mutex = new Mutex(); +let cachedToken = null; +let tokenExpiresAt = 0; +// SASE unified SD-WAN calls require a one-shot GET /sdwan/v2.1/api/profile +// immediately after every new token is acquired — otherwise subsequent +// SD-WAN calls come back with 403 (or 503 "target url lookup failed" +// depending on which edge processes the miss). See pan.dev "Unified +// SASE SD-WAN APIs" page. This flag tracks whether the priming call +// has run since the last token acquisition; it's cleared on refresh. +let saseSessionPrimed = false; + +// ────────────────────────────────────────────── +// Auth mode + base URL resolution +// ────────────────────────────────────────────── + +function resolveAuthMode() { + const raw = String(process.env.PRISMA_AUTH_MODE || AUTH_MODE_SASE).toLowerCase().trim(); + if (raw !== AUTH_MODE_SASE && raw !== AUTH_MODE_LEGACY) { + throw new Error( + `PRISMA_AUTH_MODE must be "${AUTH_MODE_SASE}" or "${AUTH_MODE_LEGACY}" (got "${raw}")`, + ); + } + return raw; +} + +function resolveBaseUrl() { + const mode = resolveAuthMode(); + if (mode === AUTH_MODE_SASE) { + return process.env.PRISMA_SASE_BASE_URL || DEFAULT_SASE_BASE_URL; + } + return process.env.PRISMA_LEGACY_BASE_URL || DEFAULT_LEGACY_BASE_URL; +} + +// ────────────────────────────────────────────── +// Axios instance — baseURL resolved at import time +// ────────────────────────────────────────────── + +export const paloAltoAxios = axios.create({ + // baseURL resolved lazily on the first request so tests can set + // PRISMA_SASE_BASE_URL / PRISMA_LEGACY_BASE_URL after import. + timeout: 20000, + headers: { + 'Accept': 'application/json', + 'Content-Type': 'application/json', + }, +}); + +// Request interceptor: (a) apply resolved baseURL on the fly if the +// caller passed a relative URL; (b) inject the correct auth header +// for the current auth mode; (c) prime the SASE unified SD-WAN +// session on first use of a new token. +paloAltoAxios.interceptors.request.use(async (cfg) => { + const mode = resolveAuthMode(); + + if (cfg.url && !/^https?:\/\//i.test(cfg.url)) { + cfg.baseURL = resolveBaseUrl(); + } + + const token = await getPrismaToken(); + + if (mode === AUTH_MODE_SASE) { + if (!cfg.headers.Authorization) { + cfg.headers.Authorization = `Bearer ${token}`; + } + // Prime the unified SD-WAN session — but skip when the current + // request IS the profile call itself (avoids infinite recursion) + // or when it's the auth POST (which uses raw axios, not this + // instance, but belt-and-braces). + if (!saseSessionPrimed && !cfg._skipSasePrime && !/\/sdwan\/v2\.1\/api\/profile$/.test(cfg.url || '')) { + try { + await primeSaseSession(token); + } catch (err) { + // Priming failed — log and continue. Downstream call will + // 403 loudly which is a better signal than swallowing here. + logger('paloalto:client', `SASE session prime failed: ${err.message}`, 'warn'); + } + } + } else { + if (!cfg.headers['x-auth-token']) { + cfg.headers['x-auth-token'] = token; + } + } + + logger('paloalto:request', `${cfg.method?.toUpperCase()} ${cfg.baseURL || ''}${cfg.url}`, 'debug'); + return cfg; +}); + +// Maximum attempts on a 429 (rate limit). One retry with backoff has +// been empirically enough in the observed Prisma tenant — the second +// attempt lands after the burst window resets. Bumping past 2 would +// stack requests behind an already-throttled endpoint and make things +// worse. `Retry-After` (seconds) is honored when present; otherwise +// exponential backoff kicks in. +const MAX_429_RETRIES = 2; +const BASE_429_BACKOFF_MS = 1500; + +paloAltoAxios.interceptors.response.use( + (response) => response, + async (error) => { + const status = error.response?.status; + + if (status === 401 && !error.config?._retry) { + logger('paloalto:client', '401 → forcing token refresh + one-shot retry', 'warn'); + try { + await getPrismaToken(true); + } catch (refreshErr) { + logger('paloalto:client', `Token refresh failed on 401: ${refreshErr.message}`, 'error'); + return Promise.reject(error); + } + + const originalRequest = error.config; + originalRequest._retry = true; + const mode = resolveAuthMode(); + if (mode === AUTH_MODE_SASE) { + originalRequest.headers.Authorization = `Bearer ${cachedToken}`; + } else { + originalRequest.headers['x-auth-token'] = cachedToken; + } + return paloAltoAxios(originalRequest); + } + + // 429 — Prisma throttles bursts of parallel metric calls. Honor + // the `Retry-After` header when present; otherwise exponential + // backoff (1.5s / 3s / 6s). Only retry idempotent verbs (GET, + // POST-with-body-that's-a-query — Prisma metric endpoints are + // read-only despite being POSTs). Capped at MAX_429_RETRIES. + if (status === 429) { + const cfg = error.config || {}; + cfg._429retries = (cfg._429retries || 0) + 1; + if (cfg._429retries <= MAX_429_RETRIES) { + const retryAfterHdr = error.response?.headers?.['retry-after']; + const retryAfterMs = retryAfterHdr + ? Math.max(0, Number(retryAfterHdr) * 1000) || null + : null; + const waitMs = retryAfterMs != null + ? retryAfterMs + : BASE_429_BACKOFF_MS * Math.pow(2, cfg._429retries - 1); + logger( + 'paloalto:client', + `429 on ${cfg.method?.toUpperCase() || 'REQ'} ${cfg.url || ''} — retry ${cfg._429retries}/${MAX_429_RETRIES} in ${waitMs}ms` + + (retryAfterMs != null ? ' (Retry-After honored)' : ''), + 'warn', + ); + await new Promise((resolve) => setTimeout(resolve, waitMs)); + return paloAltoAxios(cfg); + } + logger( + 'paloalto:client', + `429 on ${cfg.method?.toUpperCase() || 'REQ'} ${cfg.url || ''} — retries exhausted (${MAX_429_RETRIES})`, + 'warn', + ); + } + + const body = error.response?.data; + const msg = status + ? `${status} — ${typeof body === 'string' ? body : JSON.stringify(body)}` + : error.message; + logger('paloalto:error', msg, status === 429 ? 'warn' : 'error'); + return Promise.reject(error); + }, +); + +// ────────────────────────────────────────────── +// Token acquisition (dispatches on auth mode) +// ────────────────────────────────────────────── + +/** + * Return a valid Prisma auth token, refreshing if needed. Concurrent + * callers coalesce onto a single refresh through the shared mutex. + * Any refresh clears `saseSessionPrimed` so the interceptor knows to + * re-run the mandatory GET /sdwan/v2.1/api/profile priming call on + * the next SD-WAN request. + */ +export async function getPrismaToken(forceRefresh = false) { + const release = await mutex.acquire(); + try { + const now = Date.now(); + if (!forceRefresh && cachedToken && now < tokenExpiresAt) { + return cachedToken; + } + + const mode = resolveAuthMode(); + if (mode === AUTH_MODE_SASE) { + await refreshSaseToken(); + } else { + await refreshLegacyToken(); + } + saseSessionPrimed = false; + return cachedToken; + } finally { + release(); + } +} + +/** + * Mandatory session-init for the SASE unified SD-WAN surface. Per + * pan.dev's "Unified SASE SD-WAN APIs" page, this exact GET call + * MUST run immediately after obtaining a new access token — without + * it, every subsequent SD-WAN call is rejected (usually 403, but + * SASE Edge can also respond with 503 "target url lookup failed + * for proxy: null in environment prod-global" if the priming state + * is missing at the routing layer). + * + * We bypass the axios instance for this call (raw axios) to avoid + * the request interceptor's own priming check firing recursively. + * `_skipSasePrime` also protects the instance path if a caller ever + * hits the profile URL directly. + */ +async function primeSaseSession(token) { + const baseUrl = process.env.PRISMA_SASE_BASE_URL || DEFAULT_SASE_BASE_URL; + logger('paloalto:client', 'Priming SASE SD-WAN session (GET /sdwan/v2.1/api/profile)', 'debug'); + await axios.get(`${baseUrl}/sdwan/v2.1/api/profile`, { + headers: { + 'Authorization': `Bearer ${token}`, + 'Accept': 'application/json', + }, + timeout: 10000, + }); + saseSessionPrimed = true; + logger('paloalto:client', 'SASE SD-WAN session primed'); +} + +async function refreshSaseToken() { + const clientId = process.env.PRISMA_CLIENT_ID; + const clientSecret = process.env.PRISMA_CLIENT_SECRET; + const tsgId = process.env.PRISMA_TSG_ID; + const authUrl = process.env.PRISMA_AUTH_URL || DEFAULT_SASE_AUTH_URL; + + if (!clientId || !clientSecret || !tsgId) { + throw new Error( + 'PRISMA_AUTH_MODE=sase requires PRISMA_CLIENT_ID, PRISMA_CLIENT_SECRET, and PRISMA_TSG_ID.', + ); + } + + logger('paloalto:client', 'Fetching new SASE OAuth token', 'debug'); + + const basicAuth = Buffer.from(`${clientId}:${clientSecret}`).toString('base64'); + const body = new URLSearchParams({ + grant_type: 'client_credentials', + // SASE requires the TSG (Tenant Service Group) scope so the + // token is bound to the right tenant. + scope: `tsg_id:${tsgId}`, + }).toString(); + + const res = await axios.post(authUrl, body, { + headers: { + 'Authorization': `Basic ${basicAuth}`, + 'Content-Type': 'application/x-www-form-urlencoded', + }, + timeout: 10000, + }); + + const { access_token, expires_in } = res.data || {}; + if (!access_token) { + throw new Error(`SASE token response missing access_token: ${JSON.stringify(res.data)}`); + } + + cachedToken = access_token; + tokenExpiresAt = Date.now() + ((Number(expires_in) || 900) * 1000) - REFRESH_MARGIN_MS; + logger( + 'paloalto:client', + `SASE token acquired (expires in ~${Math.round((Number(expires_in) || 900) / 60)} min)`, + ); +} + +async function refreshLegacyToken() { + const email = process.env.PRISMA_EMAIL; + const password = process.env.PRISMA_PASSWORD; + const baseUrl = process.env.PRISMA_LEGACY_BASE_URL || DEFAULT_LEGACY_BASE_URL; + + if (!email || !password) { + throw new Error('PRISMA_AUTH_MODE=legacy requires PRISMA_EMAIL and PRISMA_PASSWORD.'); + } + + logger('paloalto:client', 'Fetching new legacy CloudGenix session token', 'debug'); + + const res = await axios.post( + `${baseUrl}/v2.0/api/login`, + { email, password }, + { + headers: { 'Content-Type': 'application/json' }, + timeout: 10000, + }, + ); + + const token = + res.data?.x_auth_token || + res.data?.token || + res.headers?.['x-auth-token']; + + if (!token) { + throw new Error(`Legacy login response missing token: ${JSON.stringify(res.data)}`); + } + + cachedToken = token; + // Legacy tokens don't declare expiry in the response body; treat + // as 8h with the same refresh margin. If a mid-day 401 happens + // the response interceptor still catches it and forces refresh. + tokenExpiresAt = Date.now() + (8 * 60 * 60 * 1000) - REFRESH_MARGIN_MS; + logger('paloalto:client', 'Legacy session token acquired (~8h validity)'); +} + +/** + * Test-only helper — clears the in-memory token cache so a fresh + * auth attempt happens on the next request. Not for production use + * (the mutex-guarded refresh path already handles all real cases). + */ +export function _resetPrismaAuthCache() { + cachedToken = null; + tokenExpiresAt = 0; + saseSessionPrimed = false; +} + +export default paloAltoAxios; diff --git a/integrations/paloalto/index.js b/integrations/paloalto/index.js new file mode 100644 index 0000000..9a95563 --- /dev/null +++ b/integrations/paloalto/index.js @@ -0,0 +1,29 @@ +// src/integrations/paloalto/index.js +// +// Barrel re-export for the Palo Alto Prisma SD-WAN integration. +// Consumers should import from this file rather than reaching into +// individual sub-modules — same pattern as integrations/serviceChannel/. + +export { + paloAltoAxios, + getPrismaToken, + _resetPrismaAuthCache, +} from './client.js'; + +export { + siteNameForStore, + findSdwanSiteForStore, + getAllSites, + getElementsForSite, + getWanInterfacesForSite, + _resetSitesCache, +} from './sites.js'; + +export { + LQM_METRIC_NAMES, + getHealthscore, + getLqmMetric, + getAlarms, +} from './metrics.js'; + +export { default } from './client.js'; diff --git a/integrations/paloalto/metrics.js b/integrations/paloalto/metrics.js new file mode 100644 index 0000000..63e4d83 --- /dev/null +++ b/integrations/paloalto/metrics.js @@ -0,0 +1,337 @@ +// src/integrations/paloalto/metrics.js +// +// Thin POST wrappers around Prisma SD-WAN's monitor v2.0 metrics +// API. All endpoints are POST — Prisma's monitor API always accepts +// a JSON body with `start_time` + `interval` + `metrics` + `filter`, +// even for what would idiomatically be a GET on other systems. See +// https://pan.dev/sdwan/api/metrics/ for the endpoint catalog. +// +// Endpoint / body-shape notes verified against a live SASE tenant. +// This section is the historical record of what broke and how the +// tenant's schema differs from pan.dev — read before changing any +// wrapper, because "helpful cleanup" here almost always regresses. +// +// - LQM point metrics live at a DEDICATED endpoint: +// POST /sdwan/monitor/v2.0/api/monitor/lqm_point_metrics +// NOT `sys_point_metrics` — that endpoint is for system metrics +// (CPU, Memory, Disk) and its schema rejects `filter.waninterface` +// ("is not defined in the schema"). LQM has its own endpoint. +// Metric names on the LQM endpoint carry the `PointMetric` +// suffix: `LqmLatencyPointMetric`, `LqmJitterPointMetric`, +// `LqmPktLossPointMetric` (Pkt, not Packet), `LqmMosPointMetric`. +// +// - LQM body shape on this tenant: +// `end_time` → rejected as "not defined in the schema" +// (endpoint interprets window as [start_time, now)) +// Filter shape: +// `filter.site` — ARRAY of strings. Live 400 was +// "$.filter.site: string found, array expected". +// This is OPPOSITE to sys_point_metrics, +// which wants a plain string. +// `filter.path` — array. This is the KEY THAT WORKS on +// lqm_point_metrics for scoping to +// specific circuits — even though the +// circuit ids we pass are waninterface +// ids from /sites/{id}/waninterfaces. +// The LIVEcommunity working example uses +// this exact shape: +// filter: { site: [id], path: [wi_id] } +// Both `wan_interfaces` (plural + underscore) +// and `waninterface` (sys_point_metrics +// variant) are rejected as "not defined in +// the schema" on this tenant. +// No `elements` filter (that's a sys_point_metrics quirk). +// +// - `metrics[].statistics` is a PLURAL array (`["average"]` / +// `["max"]`), not the singular `statistic`. +// +// - `metrics[].unit` is case-sensitive: `milliseconds`, `Percentage`, +// `gauge`, `count`. +// +// - Healthscore on this tenant: v2.0 aiops/health is a DEAD END. +// Confirmed via prismaProbe try-shapes: +// v2.0 `metrics: [...]` → rejected ("not defined") +// v2.0 with `end_time` removed → 400 "end_time is required" +// v2.0 with `end_time` added + → same "$.metrics: not defined" +// v2.1 aiops/aggregates → requires app_id + aggregates +// v2.6 unified /monitor/metrics → ✅ 200 OK (only winner) +// We use `POST /sdwan/monitor/v2.6/api/monitor/metrics` with the +// standard `metrics: [{Healthscore}]` + `filter: {site: [id]}`. +// This endpoint accepts filter.site as an ARRAY and returns a +// per-site healthscore in the response. +// +// - `interval` must be one of Prisma's canonical bucket sizes: +// `10sec`, `1min`, `5min`, `1hour`, `1day`. `15min` etc. +// yield HTTP 400 with SCHEMA_CHECK_FAIL on `$.interval`. +// +// - Alarms live at `POST /sdwan/v3.7/api/events/query`, NOT +// `/sdwan/monitor/v2.0/api/monitor/alarms` (that endpoint +// 404s "ROUTE_NOT_FOUND" on the observed tenant). +// +// All wrappers return `null` on any failure and log the error with +// a preview of the response body so shape drift is diagnosable +// without cranking LOG_LEVEL=debug. Higher layers absorb per-metric +// failures via `Promise.allSettled`. + +import { paloAltoAxios } from './client.js'; +import { logger } from '../../utils/logger.js'; + +// Prisma metric-name catalog for the LQM (Link Quality Monitoring) +// bucket. Short JS keys → { name, unit } for the API body. +// +// Names carry the `PointMetric` suffix because this maps to the +// dedicated `lqm_point_metrics` endpoint (NOT `sys_point_metrics`). +// The unit strings are verified via 400 SCHEMA_CHECK_FAIL responses +// when they drift and are case-sensitive: +// - `milliseconds` (lowercase) → latency, jitter +// - `percentage` (lowercase — NOT `Percentage`, verified via +// prismaProbe try-shapes lqm-loss on 2026-07-09 +// which returned 400 METRIC_UNIT_NOT_SUPPORTED for +// every capitalized/alternate variant) → loss +// - `count` → mos +// If Prisma ever rejects a unit here, re-run the matrix probe: +// `node scripts/prismaProbe.js try-shapes lqm-loss `. +export const LQM_METRIC_NAMES = Object.freeze({ + latency: { name: 'LqmLatencyPointMetric', unit: 'milliseconds' }, + jitter: { name: 'LqmJitterPointMetric', unit: 'milliseconds' }, + loss: { name: 'LqmPktLossPointMetric', unit: 'percentage' }, + mos: { name: 'LqmMosPointMetric', unit: 'count' }, +}); + +// ────────────────────────────────────────────── +// URL helpers — SASE and legacy have different prefixes +// ────────────────────────────────────────────── + +function isSase() { + return String(process.env.PRISMA_AUTH_MODE || 'sase').toLowerCase().trim() === 'sase'; +} + +function monitorPath(endpoint) { + return isSase() + ? `/sdwan/monitor/v2.0/api/monitor/${endpoint}` + : `/v2.0/api/monitor/${endpoint}`; +} + +// v2.6 unified monitor/metrics endpoint. The observed tenant only +// serves healthscore correctly here — v2.0 aiops/health is a dead +// end (see the top-of-file schema notes). +function metricsV26Path() { + return isSase() + ? '/sdwan/monitor/v2.6/api/monitor/metrics' + : '/v2.6/api/monitor/metrics'; +} + +// Events endpoint (SASE v3.7 is the current top of the docs). Used +// for alarm retrieval — the older `/monitor/alarms` path 404s on +// the SASE surface. +function eventsQueryPath() { + return isSase() + ? '/sdwan/v3.7/api/events/query' + : '/v3.7/api/events/query'; +} + +// ────────────────────────────────────────────── +// Time-window helpers +// ────────────────────────────────────────────── + +function windowStart(minutes) { + return new Date(Date.now() - minutes * 60 * 1000).toISOString(); +} + +function nowIso() { + return new Date().toISOString(); +} + +// Prisma's valid `interval` enum (verified via HTTP 400 error message). +// `15min`, `30min` etc. are NOT accepted — snap up to the next valid +// bucket (1hour) rather than silently down-sampling. +function pickInterval(minutes) { + if (minutes <= 1) return '1min'; + if (minutes <= 5) return '5min'; + if (minutes < 60) return '5min'; // 15-minute default now snaps to 5min buckets + if (minutes <= 60) return '1hour'; + return '1day'; +} + +// Small helper that logs a failed metric call with a response-body +// preview AND (on 4xx) the request body that Prisma rejected — the +// most common failure mode is a metric name or filter key drifting +// on the tenant, and seeing both sides in the same log line makes +// the schema mismatch obvious without cranking LOG_LEVEL=debug. +function logMetricFailure(scope, err) { + const status = err.response?.status; + const respPreview = err.response?.data + ? JSON.stringify(err.response.data).slice(0, 300) + : ''; + const isSchemaError = status && status >= 400 && status < 500; + const reqPreview = isSchemaError && err.config?.data + ? String(err.config.data).slice(0, 500) + : ''; + const parts = [ + `${scope} failed: ${err.message}`, + status ? `(HTTP ${status})` : '', + respPreview ? `— resp: ${respPreview}` : '', + reqPreview ? `— req: ${reqPreview}` : '', + ].filter(Boolean); + logger('paloalto:metrics', parts.join(' '), 'warn'); +} + +// ────────────────────────────────────────────── +// Healthscore (site-level roll-up 0-100) +// ────────────────────────────────────────────── + +/** + * Site healthscore (0-100) over the given window. Uses the v2.6 + * unified monitor/metrics endpoint — the only healthscore endpoint + * that returned 200 on this tenant (verified via prismaProbe + * try-shapes; v2.0 aiops/health and v2.1 aiops/aggregates both + * dead-end on schema errors this tenant enforces). + * + * The v2.6 endpoint accepts: + * - `metrics: [{ name: 'Healthscore', statistics: ['max'], unit: 'gauge' }]` + * - `view: {}` + * - `filter: { site: [siteId] }` — scoped server-side (unlike + * v2.0 which rejects any filter) + * - `interval` + `start_time` (no `end_time`) + * + * @param {string} siteId + * @param {number} windowMinutes default 15 + * @returns {Promise} raw response or null on failure + */ +export async function getHealthscore(siteId, windowMinutes = 15) { + if (!siteId) return null; + try { + const res = await paloAltoAxios.post(metricsV26Path(), { + start_time: windowStart(windowMinutes), + interval: pickInterval(windowMinutes), + metrics: [{ + name: 'Healthscore', + statistics: ['max'], + unit: 'gauge', + }], + view: {}, + filter: { site: [String(siteId)] }, + }); + return res.data || null; + } catch (err) { + logMetricFailure(`getHealthscore(${siteId})`, err); + return null; + } +} + +// ────────────────────────────────────────────── +// LQM point metrics (per-path latency / jitter / loss / MOS) +// ────────────────────────────────────────────── + +/** + * Fetch one of the four LQM point metrics over the given window. + * `metricKey` is one of the keys of LQM_METRIC_NAMES ('latency', + * 'jitter', 'loss', 'mos'). + * + * Uses the DEDICATED `lqm_point_metrics` endpoint. `sys_point_metrics` + * is a different endpoint for CPU/Memory/Disk system metrics and its + * schema rejects `filter.wan_interfaces` — that was the bug that led + * to the endpoint switch. + * + * Filter shape verified against live 400s on this tenant + the + * pan.dev LIVEcommunity #1235108 working example: + * - `filter.site` — ARRAY (opposite of sys_point_metrics, which + * wants a string). Live 400 was + * "$.filter.site: string found, array expected". + * - `filter.path` — array of waninterface ids. Yes, the key is + * "path" — NOT `wan_interfaces` (rejected as + * "not defined in the schema" on this tenant), + * NOT `waninterface` (that's the sys_point_metrics + * key). Prisma's LQM data model treats each + * circuit as a "path". + * + * Missing `waninterfaceIds` returns null (with a log line) rather + * than firing a doomed request that would come back empty. + * + * @param {string} siteId + * @param {string[]} waninterfaceIds one or more WAN interface ids + * @param {'latency'|'jitter'|'loss'|'mos'} metricKey + * @param {number} windowMinutes default 5 + * @returns {Promise} + */ +export async function getLqmMetric(siteId, waninterfaceIds, metricKey, windowMinutes = 5) { + const spec = LQM_METRIC_NAMES[metricKey]; + if (!spec) { + throw new Error(`Unknown LQM metric key "${metricKey}" (expected: ${Object.keys(LQM_METRIC_NAMES).join(', ')})`); + } + if (!siteId) return null; + if (!Array.isArray(waninterfaceIds) || waninterfaceIds.length === 0) { + logger('paloalto:metrics', `getLqmMetric(${metricKey}): no waninterfaceIds — skipping`, 'debug'); + return null; + } + + try { + const res = await paloAltoAxios.post(monitorPath('lqm_point_metrics'), { + // NOTE: no `end_time` — the observed tenant's lqm_point_metrics + // schema rejects it with 400 "$.end_time: is not defined in the + // schema and the schema does not allow additional properties". + // The endpoint interprets the window as [start_time, now). + start_time: windowStart(windowMinutes), + interval: pickInterval(windowMinutes), + metrics: [{ + name: spec.name, + statistics: ['average'], + unit: spec.unit, + }], + view: {}, + filter: { + site: [String(siteId)], + path: waninterfaceIds, + }, + }); + return res.data || null; + } catch (err) { + logMetricFailure(`getLqmMetric(${siteId}, ${metricKey})`, err); + return null; + } +} + +// ────────────────────────────────────────────── +// Alarms (via the events/query endpoint) +// ────────────────────────────────────────────── + +/** + * Recent alarms for the site. Uses the SASE events endpoint + * (`POST /sdwan/v3.7/api/events/query`) — the older + * `/monitor/v2.0/api/monitor/alarms` path 404s on this tenant + * ("ROUTE_NOT_FOUND_0001"). + * + * Body shape mirrors the Prisma ServiceNow CloudBlade example + * (pan.dev + PA integration guide): `limit` is an OBJECT with + * `count`/`sort_on`/`sort_order`, `query` carries `site`/`type`, + * `severity` is a top-level array, `start_time` is ISO. + * Additionally we filter `type: ['alarm']` so we don't get + * informational events mixed into the alarm count. + * + * @param {string} siteId + * @param {number} windowMinutes default 60 + * @returns {Promise} + */ +export async function getAlarms(siteId, windowMinutes = 60) { + if (!siteId) return null; + + try { + const res = await paloAltoAxios.post(eventsQueryPath(), { + limit: { + count: 100, + sort_on: 'time', + sort_order: 'descending', + }, + query: { + site: [String(siteId)], + type: ['alarm'], + }, + severity: ['critical', 'major', 'minor'], + start_time: windowStart(windowMinutes), + }); + return res.data || null; + } catch (err) { + logMetricFailure(`getAlarms(${siteId})`, err); + return null; + } +} diff --git a/integrations/paloalto/sites.js b/integrations/paloalto/sites.js new file mode 100644 index 0000000..cdf34f9 --- /dev/null +++ b/integrations/paloalto/sites.js @@ -0,0 +1,431 @@ +// src/integrations/paloalto/sites.js +// +// Site + element discovery for Prisma SD-WAN. Prisma's data model +// is: tenant → site → element(s) → interface(s)/path(s). A "site" is +// what maps 1:1 to a store; an "element" is a physical/virtual +// Prisma appliance at that site. +// +// AE convention (confirmed by operator): site names are +// `CG${storeNum.padStart(5, '0')}` — e.g. store 782 → `CG00782`, +// store 2477 → `CG02477`, store 305 → `CG00305`. Exact match, no +// fuzzy scoring needed. This is materially simpler than the Meraki +// resolver (which has to substring-search). +// +// Cache strategy: +// - Sites list: 4-hour TTL (same as Meraki networks cache). Sites +// don't come and go often; a store add/remove is a Prisma-side +// lifecycle event that we tolerate up to 4 hours of staleness +// on. `refreshSitesCache()` re-pulls on demand. +// - Elements per site: cached alongside the site the first time +// they're asked for. Same 4-hour TTL. Elements at a site can +// change (RMA replacement), but this is rare and the same +// staleness bound applies. +// +// Read-only. The client only calls GET here; there's no mutation +// surface exposed. + +import { paloAltoAxios } from './client.js'; +import { logger } from '../../utils/logger.js'; + +const CACHE_TTL_MS = 4 * 60 * 60 * 1000; + +// Sites list cache (one entry for the whole tenant). +let sitesCache = null; +let sitesCacheAt = 0; + +// Tenant-wide elements cache. Rationale (verified against live +// tenant): the `?site_id=X` GET filter on /elements is NOT honored +// server-side — Prisma returns the full inventory (1229 elements in +// the observed tenant) regardless of the query param, and we filter +// client-side. Making one tenant-wide fetch and indexing by site_id +// in-memory eliminates 1200+ redundant records per subsequent call +// and cuts /voicediag's Prisma call count by ~90%. +let elementsCache = null; // { elementsBySite: Map, fetchedAt } + +// Per-site waninterfaces cache (Map). +// This one stays per-site because /sites/{siteId}/waninterfaces IS +// properly scoped server-side — no wasteful over-fetch. +const waninterfacesCache = new Map(); + +// ────────────────────────────────────────────── +// Site resolution +// ────────────────────────────────────────────── + +/** + * Convert a 2-4 digit store number into its expected Prisma site + * name using the `CG${pad5}` convention. + * + * @param {string|number} storeNum + * @returns {string} e.g. 'CG00782' + */ +export function siteNameForStore(storeNum) { + const digits = String(storeNum ?? '').match(/\d+/)?.[0] || ''; + if (!digits) { + throw new Error(`siteNameForStore: no digits in "${storeNum}"`); + } + const padded = digits.padStart(5, '0').slice(-5); + return `CG${padded}`; +} + +/** + * Return all Prisma SD-WAN sites in the tenant, refreshing the + * cache when stale. Uses the GET-based list endpoint: + * + * GET /sdwan/v4.13/api/sites (SASE unified — "Get Sites Of Tenant") + * GET /v4.13/api/sites (legacy) + * + * Why GET instead of the sibling `POST /sites/query`: + * + * The observed SASE tenant (2026-07-08) runs a schema variant of + * `POST /sites/query` where the request-body validator rejects + * fields with cryptic type-mismatch errors — first `getDeleted` + * ("expected boolean, got object"), then `total_count` ("expected + * long, got object"), and likely more if we probed further. The + * SASE proxy appears to auto-inject fields with `{}` defaults into + * the body before schema validation, so ANY minimal body triggers + * a different validation failure on some auto-injected field. + * + * `GET /sites` sidesteps the entire body-schema minefield: no body + * = no auto-injection = no drifting field types to guess. The + * endpoint returns the same `{ items: [...] }` shape and is + * documented on pan.dev as "Get Sites Of Tenant (v4.13)". + * + * Pagination: the GET endpoint supports `?limit=` and cursor + * follow-up if Prisma paginates its response. We handle both + * `items[]` on a single page and any `next_query` echo (defensively). + */ +export async function getAllSites(forceRefresh = false) { + const now = Date.now(); + if (!forceRefresh && sitesCache && now - sitesCacheAt < CACHE_TTL_MS) { + return sitesCache; + } + + const path = sitesListPath(); + logger('paloalto:sites', `Refreshing sites cache from GET ${path}`, 'debug'); + + try { + const allItems = await fetchAllSitesGet(path); + + sitesCache = allItems.map((s) => ({ + id: s.id, + name: s.name, + description: s.description || '', + tenant_id: s.tenant_id || null, + raw: s, + })); + sitesCacheAt = now; + + if (sitesCache.length === 0) { + logger('paloalto:sites', `Cached 0 sites (empty tenant or response-shape drift — enable LOG_LEVEL=debug to see the raw first-page shape).`, 'warn'); + } else { + // Sample the first 3 site names so an operator scanning the + // logs can immediately eyeball whether their target site name + // matches the naming convention Prisma is actually using. + const sample = sitesCache.slice(0, 3).map((s) => s.name).join(', '); + logger( + 'paloalto:sites', + `Cached ${sitesCache.length} sites (sample: ${sample}${sitesCache.length > 3 ? ', …' : ''})`, + ); + } + return sitesCache; + } catch (err) { + logger('paloalto:sites', `Failed to fetch sites: ${err.message}`, 'error'); + // On failure, keep any stale cache (better than nothing) but + // return an empty array if we've never fetched successfully. + return sitesCache || []; + } +} + +/** + * Fetch every site via `GET /sdwan/v4.13/api/sites`, following any + * cursor pagination Prisma returns. + * + * The GET endpoint's response shape mirrors POST /query: + * { items: [...], next_query?: {...} } + * + * Some tenants return a single-shot list (all sites in one response, + * no `next_query`). Others paginate — in which case we send the + * `next_query` object back as a `?query=` query + * param (Prisma's GET-side pagination convention) or via a subsequent + * POST hit. We try the "single response" path first and only + * paginate if `next_query` is populated. + * + * Defenses: + * - Request `?limit=1000` up-front to maximise the first-page size + * (some tenants default to 100). + * - Follow `next_query` when present. Empty / null / undefined / + * empty-object all count as "no more pages" (same rules as the + * POST paginator). + * - MAX_PAGES safety cap against a runaway cursor. + */ +async function fetchAllSitesGet(basePath) { + const PAGE_LIMIT = 1000; + const MAX_PAGES = 50; + const all = []; + + let requestParams = { limit: PAGE_LIMIT }; + let page = 0; + + while (page < MAX_PAGES) { + page += 1; + logger( + 'paloalto:sites', + `Requesting page ${page}: GET ${basePath} params=${JSON.stringify(requestParams).slice(0, 200)}`, + 'debug', + ); + const res = await paloAltoAxios.get(basePath, { params: requestParams }); + + let items = res.data?.items || res.data?.data; + if (!items && Array.isArray(res.data)) items = res.data; + if (!items) items = []; + if (!Array.isArray(items)) { + const shapeHint = res.data && typeof res.data === 'object' + ? `keys=[${Object.keys(res.data).join(', ')}]` + : `type=${typeof res.data}`; + throw new Error(`Unexpected sites response shape on page ${page} (${shapeHint})`); + } + + all.push(...items); + logger('paloalto:sites', `Page ${page}: got ${items.length} sites (running total ${all.length})`, 'debug'); + + // Termination: no cursor → done. GET endpoints without cursor + // return the full list in one response for most tenants. + const nextQuery = res.data?.next_query; + const hasCursor = nextQuery + && typeof nextQuery === 'object' + && Object.keys(nextQuery).length > 0; + + if (!hasCursor) break; + + // Pass Prisma's own cursor state onto the next GET as query + // params. The most portable representation is `?query=` + // — Prisma's GET-side pagination convention — but some + // tenants also honor spreading the cursor fields directly. + // We spread first (more common); if that ever breaks, fall + // back to the JSON-encoded param. + requestParams = { ...nextQuery, limit: PAGE_LIMIT }; + } + + if (page >= MAX_PAGES) { + logger( + 'paloalto:sites', + `Pagination hit MAX_PAGES=${MAX_PAGES} at ${all.length} sites — cursor may be looping. Investigate.`, + 'warn', + ); + } + + return all; +} + +/** + * Given a store number (2-4 digits, or a string containing them), + * return the corresponding Prisma site or null if not found. + * Exact match on `CG${pad5}` — the only convention AE uses. + * + * @param {string|number} storeNum + * @returns {Promise<{id:string,name:string,description:string,tenant_id:string|null}|null>} + */ +export async function findSdwanSiteForStore(storeNum) { + let expectedName; + try { + expectedName = siteNameForStore(storeNum); + } catch (err) { + logger('paloalto:sites', `Bad store number "${storeNum}": ${err.message}`, 'warn'); + return null; + } + + const sites = await getAllSites(); + const match = sites.find((s) => s.name === expectedName); + if (!match) { + // Warn (not debug) so an operator can see mismatches in + // default-level logs. Include the nearest names in the tenant so + // a naming-convention drift ("CG782" vs "CG00782") is spotted + // without hunting through the full site list. + const nearby = sites + .filter((s) => (s.name || '').includes(String(storeNum))) + .slice(0, 5) + .map((s) => s.name); + const nearbyHint = nearby.length > 0 ? ` — closest matches: ${nearby.join(', ')}` : ''; + logger( + 'paloalto:sites', + `No Prisma site matches "${expectedName}" (store ${storeNum}) among ${sites.length} sites${nearbyHint}`, + 'warn', + ); + return null; + } + logger('paloalto:sites', `Store ${storeNum} → site ${match.name} (${match.id})`); + return match; +} + +// ────────────────────────────────────────────── +// Elements (appliances) per site +// ────────────────────────────────────────────── + +/** + * Return the list of Prisma appliances (elements) at a site. + * + * Fetches the ENTIRE tenant inventory once (cached 4h) and indexes + * by `site_id` so per-site lookups are O(1) with zero network cost + * on cache hit. This is the correct pattern for Prisma because the + * `?site_id=X` server-side filter on /elements is silently ignored + * (verified live: a filtered request returned 1229 elements for a + * site that owns 1). + * + * GET /sdwan/v3.1/api/elements (SASE unified — tenant-wide) + * GET /v3.1/api/elements (legacy) + * + * @param {string} siteId + * @returns {Promise>} + */ +export async function getElementsForSite(siteId, forceRefresh = false) { + if (!siteId) return []; + + const map = await getElementsIndex(forceRefresh); + const rows = map.get(siteId) || []; + logger('paloalto:sites', `Site ${siteId}: ${rows.length} element(s) from tenant-wide cache`, 'debug'); + return rows; +} + +/** + * Return (fetching if stale) the tenant-wide index of elements + * keyed by site_id. First caller pays the network cost + build; all + * subsequent callers within the 4h TTL get instant Map lookup. + */ +async function getElementsIndex(forceRefresh = false) { + const now = Date.now(); + if (!forceRefresh && elementsCache && now - elementsCache.fetchedAt < CACHE_TTL_MS) { + return elementsCache.elementsBySite; + } + + const path = elementsPath(); + logger('paloalto:sites', `Refreshing tenant-wide elements from ${path}`, 'debug'); + + try { + const res = await paloAltoAxios.get(path); + const items = res.data?.items || res.data?.data || []; + if (!Array.isArray(items)) { + throw new Error(`Unexpected elements response shape: ${JSON.stringify(res.data)?.slice(0, 200)}`); + } + const elementsBySite = new Map(); + for (const e of items) { + if (!e.site_id) continue; + const row = { + id: e.id, + name: e.name || null, + model: e.model_name || e.model || null, + serial_number: e.serial_number || null, + connected: e.connected === true, + site_id: e.site_id, + raw: e, + }; + const bucket = elementsBySite.get(e.site_id) || []; + bucket.push(row); + elementsBySite.set(e.site_id, bucket); + } + elementsCache = { elementsBySite, fetchedAt: now }; + logger( + 'paloalto:sites', + `Cached ${items.length} elements across ${elementsBySite.size} site(s)`, + ); + return elementsBySite; + } catch (err) { + logger('paloalto:sites', `Failed to fetch tenant elements: ${err.message}`, 'error'); + return elementsCache?.elementsBySite || new Map(); + } +} + +/** + * Return the list of WAN interfaces (per-site circuits) for a site. + * Cached per site for 4h. WAN interfaces are the "path" identifiers + * that LQM metrics filter by — without this list, per-path latency / + * jitter / loss / MOS queries have no waninterface ids to feed into + * the metric filter and come back empty. + * + * GET /sdwan/v2.10/api/sites/{siteId}/waninterfaces (SASE unified) + * GET /v2.10/api/sites/{siteId}/waninterfaces (legacy) + * + * @param {string} siteId + * @returns {Promise>} + */ +export async function getWanInterfacesForSite(siteId, forceRefresh = false) { + if (!siteId) return []; + const now = Date.now(); + const cached = waninterfacesCache.get(siteId); + if (!forceRefresh && cached && now - cached.fetchedAt < CACHE_TTL_MS) { + return cached.waninterfaces; + } + + const path = waninterfacesPath(siteId); + logger('paloalto:sites', `Refreshing waninterfaces for site ${siteId} from ${path}`, 'debug'); + + try { + const res = await paloAltoAxios.get(path); + const items = res.data?.items || res.data?.data || []; + if (!Array.isArray(items)) { + throw new Error(`Unexpected waninterfaces response shape: ${JSON.stringify(res.data)?.slice(0, 200)}`); + } + const waninterfaces = items.map((w) => ({ + id: w.id, + name: w.name || w.description || null, + // Defensive parsing: on some tenant schema variants the + // waninterface config API doesn't return `admin_up` at all, + // in which case a strict `=== true` check silently reports + // every circuit as DOWN — a misleading false positive + // (verified live: 3 circuits all showed ❌ DOWN when the + // API returned no admin_up field). Treat missing / non-bool + // as null (unknown) so the check + renderer show "?" rather + // than red-crossing a healthy circuit. + adminUp: typeof w.admin_up === 'boolean' ? w.admin_up : null, + wanNetworkId: w.wan_network_id || null, + wanNetworkName: w.wan_network_name || null, + // `used_for`: 'primary' | 'secondary' | ... Also useful as a + // rough "transport type" label until we wire wan_networks + // config in a phase 2. + usedFor: w.used_for || null, + bwConfigMode: w.bw_config_mode || null, + raw: w, + })); + waninterfacesCache.set(siteId, { waninterfaces, fetchedAt: now }); + logger('paloalto:sites', `Site ${siteId}: ${waninterfaces.length} waninterface(s)`); + return waninterfaces; + } catch (err) { + logger('paloalto:sites', `Failed to fetch waninterfaces for site ${siteId}: ${err.message}`, 'error'); + return cached?.waninterfaces || []; + } +} + +// ────────────────────────────────────────────── +// Path helpers — SASE vs legacy have different URL prefixes +// ────────────────────────────────────────────── + +function isSase() { + return String(process.env.PRISMA_AUTH_MODE || 'sase').toLowerCase().trim() === 'sase'; +} + +function sitesListPath() { + return isSase() + ? '/sdwan/v4.13/api/sites' + : '/v4.13/api/sites'; +} + +function elementsPath() { + return isSase() + ? '/sdwan/v3.1/api/elements' + : '/v3.1/api/elements'; +} + +function waninterfacesPath(siteId) { + return isSase() + ? `/sdwan/v2.10/api/sites/${siteId}/waninterfaces` + : `/v2.10/api/sites/${siteId}/waninterfaces`; +} + +/** + * Test-only — clears all caches so a fresh fetch happens next call. + */ +export function _resetSitesCache() { + sitesCache = null; + sitesCacheAt = 0; + elementsCache = null; + waninterfacesCache.clear(); +} diff --git a/package.json b/package.json index 22da8f3..be8a14a 100644 --- a/package.json +++ b/package.json @@ -15,6 +15,7 @@ "docker:down": "docker compose down", "docker:logs": "docker compose logs -f", "package:relay": "bash dect-relay-agent/bundle.sh", + "prisma:probe": "node scripts/prismaProbe.js", "test": "node --test tests/*.test.js" }, "dependencies": { diff --git a/scripts/prismaProbe.js b/scripts/prismaProbe.js new file mode 100644 index 0000000..af312b1 --- /dev/null +++ b/scripts/prismaProbe.js @@ -0,0 +1,802 @@ +#!/usr/bin/env node +/** + * Prisma SD-WAN API probe — a debugging harness that lets you fire + * individual Prisma API calls against a live tenant without going + * through `/phonestatus` or `/voicediag`. This is the tool for + * discovering the tenant's actual schema when documented shapes + * (pan.dev + LIVEcommunity examples) don't match. + * + * Design goals: + * - Zero side effects. Never mutates Prisma state. + * - Reuses the production axios client (same auth, retry, session + * priming). So a shape that works here will work in-band. + * - Prints BOTH the request body and the response body verbatim + * on failure, which is the whole point — you need to see the + * exact SCHEMA_CHECK_FAIL field to know what to change. + * - `try-shapes` mode iterates a curated list of body variants + * against a single endpoint and reports which pass — much + * faster than one-shot testing when you don't know the tenant's + * accepted shape. + * + * ─── Usage ───────────────────────────────────────────────────────── + * + * node scripts/prismaProbe.js [args] [flags] + * + * Subcommands: + * + * discover + * Full store discovery. Resolves store → site, then lists + * elements + waninterfaces at the site. This is what the bot + * does before firing metrics. + * + * site + * Just resolve store number → Prisma site (no per-site drilldowns). + * + * elements + * List elements at a site. + * + * waninterfaces + * List waninterfaces (circuits) at a site. + * + * health + * Fetch healthscore via the production `getHealthscore` wrapper. + * Shows what the bot would send; use `try-shapes health` to + * experiment with alternative body shapes. + * + * lqm [--metric latency|jitter|loss|mos] + * Fetch a single LQM metric via the production `getLqmMetric` + * wrapper. `--metric` defaults to `latency`. + * + * alarms [--window ] + * Fetch alarms via the production `getAlarms` wrapper. Default + * window is 60 minutes. + * + * raw [--body '{"json":"body"}'] + * Send an arbitrary request through the authenticated client. + * Useful for testing endpoints we don't yet wrap (e.g. + * `/sdwan/monitor/v2.5/api/monitor/metrics`). + * + * try-shapes health + * try-shapes lqm + * try-shapes lqm-latency + * try-shapes lqm-jitter + * try-shapes lqm-loss + * try-shapes lqm-mos + * Combinatorial shape testing: fire N candidate request bodies + * against the given endpoint and print which pass, which fail, + * and — for failures — the exact SCHEMA_CHECK_FAIL field. + * `lqm-loss` / `lqm-mos` sweep {metric name × unit} combos when + * the metric's identifier is wrong (400 METRIC_UNIT_NOT_SUPPORTED + * or METRIC_NOT_FOUND). + * + * Global flags: + * --json Emit JSON output instead of pretty-printed. + * --show-body Print the request body on 2xx as well as 4xx. + * --quiet Suppress the axios request log line. + * --help, -h Show this help. + * + * Environment: + * Reads the same `.env` as the bot (PRISMA_AUTH_MODE, PRISMA_CLIENT_ID, + * PRISMA_CLIENT_SECRET, PRISMA_TSG_ID, PRISMA_SASE_BASE_URL, + * PRISMA_AUTH_URL, and/or PRISMA_EMAIL / PRISMA_PASSWORD / + * PRISMA_LEGACY_BASE_URL for legacy auth). + * + * Examples: + * node scripts/prismaProbe.js discover 782 + * node scripts/prismaProbe.js health 16158173173100144 + * node scripts/prismaProbe.js lqm 16158173173100144 16158173176610209 --metric loss + * node scripts/prismaProbe.js try-shapes health 16158173173100144 + * node scripts/prismaProbe.js try-shapes lqm 16158173173100144 16158173176610209,1666974885552003096 + * node scripts/prismaProbe.js raw POST /sdwan/v3.7/api/events/query --body '{"limit":{"count":5}}' + */ + +import 'dotenv/config'; +import { + paloAltoAxios, + findSdwanSiteForStore, + getElementsForSite, + getWanInterfacesForSite, + getHealthscore, + getLqmMetric, + getAlarms, + LQM_METRIC_NAMES, +} from '../integrations/paloalto/index.js'; + +// ─── Arg parsing ──────────────────────────────────────────────────── + +function parseArgs(argv) { + const args = { _: [], flags: {} }; + const rest = argv.slice(2); + for (let i = 0; i < rest.length; i += 1) { + const a = rest[i]; + if (a === '--help' || a === '-h') { args.flags.help = true; continue; } + if (a === '--json') { args.flags.json = true; continue; } + if (a === '--show-body') { args.flags.showBody = true; continue; } + if (a === '--quiet') { args.flags.quiet = true; continue; } + if (a.startsWith('--')) { + const key = a.slice(2); + const next = rest[i + 1]; + if (next !== undefined && !next.startsWith('--')) { + args.flags[key] = next; + i += 1; + } else { + args.flags[key] = true; + } + continue; + } + args._.push(a); + } + return args; +} + +// ─── Small output helpers ─────────────────────────────────────────── + +const ICON_OK = '✅'; +const ICON_ERR = '❌'; +const ICON_INFO = 'ℹ️ '; +const ICON_WAIT = '⋯ '; + +function pretty(obj) { + return JSON.stringify(obj, null, 2); +} + +function shortJson(obj, max = 400) { + const s = JSON.stringify(obj); + return s.length > max ? `${s.slice(0, max)}…(+${s.length - max} chars)` : s; +} + +function heading(text) { + const bar = '─'.repeat(Math.max(4, text.length + 2)); + console.log(`\n${bar}\n ${text}\n${bar}`); +} + +function ok(msg) { console.log(`${ICON_OK} ${msg}`); } +function bad(msg) { console.log(`${ICON_ERR} ${msg}`); } +function info(msg) { console.log(`${ICON_INFO} ${msg}`); } + +/** + * Fire a raw request through the authenticated client and return a + * uniform verdict object. Never throws — all axios failures are + * caught and translated so the caller can render them in one style. + */ +async function fire({ method, url, body }) { + const startedAt = Date.now(); + try { + const res = await paloAltoAxios.request({ + method, + url, + data: body, + }); + return { + ok: true, + status: res.status, + elapsedMs: Date.now() - startedAt, + request: { method, url, body }, + response: res.data, + }; + } catch (err) { + return { + ok: false, + status: err.response?.status || 0, + elapsedMs: Date.now() - startedAt, + request: { method, url, body }, + response: err.response?.data || null, + errorMessage: err.message, + }; + } +} + +/** + * Extract the first SCHEMA_CHECK_FAIL message from a Prisma 400 + * body (`_error` array). Returns null on other error shapes so the + * caller falls back to shortJson(). + */ +function extractSchemaError(body) { + if (!body || typeof body !== 'object') return null; + const errs = body._error; + if (!Array.isArray(errs) || errs.length === 0) return null; + const first = errs[0]; + if (!first || typeof first !== 'object') return null; + return `${first.code || 'ERROR'}: ${first.message || ''}`.trim(); +} + +function printVerdict(verdict, { json, showBody }) { + if (json) { + console.log(pretty(verdict)); + return; + } + + const { ok: pass, status, elapsedMs, request, response, errorMessage } = verdict; + const icon = pass ? ICON_OK : ICON_ERR; + console.log(`${icon} ${request.method} ${request.url} → ${status || 'network fail'} (${elapsedMs}ms)`); + + const shouldShowRequest = !pass || showBody; + if (shouldShowRequest && request.body !== undefined) { + console.log(' request body:'); + console.log(' ' + pretty(request.body).replaceAll('\n', '\n ')); + } + + if (!pass) { + const schemaMsg = extractSchemaError(response); + if (schemaMsg) { + console.log(` ${ICON_ERR} ${schemaMsg}`); + } else if (errorMessage) { + console.log(` ${ICON_ERR} ${errorMessage}`); + } + } + + if (response) { + console.log(' response body:'); + console.log(' ' + pretty(response).replaceAll('\n', '\n ')); + } +} + +// ─── Subcommand implementations ───────────────────────────────────── + +async function cmdDiscover(args) { + const storeNum = args._[1]; + if (!storeNum) throw new Error('usage: discover '); + heading(`Discover store ${storeNum}`); + const site = await findSdwanSiteForStore(storeNum); + if (!site) { bad(`No Prisma site for store ${storeNum}`); return; } + ok(`site: ${site.name} (${site.id})`); + info(`description: ${site.description || '(none)'}`); + + heading('Elements at this site'); + const elements = await getElementsForSite(site.id); + if (!elements || elements.length === 0) { + bad('no elements at this site'); + } else { + for (const e of elements) { + console.log(` - ${e.name || e.id} (${e.id}) model=${e.model || '?'} connected=${e.connected}`); + } + } + + heading('WAN interfaces (circuits) at this site'); + const wans = await getWanInterfacesForSite(site.id); + if (!wans || wans.length === 0) { + bad('no waninterfaces at this site'); + } else { + for (const w of wans) { + const admin = w.adminUp === null ? '?' : (w.adminUp ? 'up' : 'down'); + console.log(` - ${w.name || w.id} (${w.id}) usedFor=${w.usedFor || '?'} adminUp=${admin}`); + } + } + + info(`Use these ids for follow-up calls:`); + console.log(` siteId = ${site.id}`); + console.log(` waninterfaceIds = ${(wans || []).map((w) => w.id).join(',')}`); +} + +async function cmdSite(args) { + const storeNum = args._[1]; + if (!storeNum) throw new Error('usage: site '); + const site = await findSdwanSiteForStore(storeNum); + if (!site) { bad(`No Prisma site for store ${storeNum}`); return; } + ok(`store ${storeNum} → ${site.name} (${site.id})`); +} + +async function cmdElements(args) { + const siteId = args._[1]; + if (!siteId) throw new Error('usage: elements '); + const els = await getElementsForSite(siteId); + console.log(pretty(els)); +} + +async function cmdWaninterfaces(args) { + const siteId = args._[1]; + if (!siteId) throw new Error('usage: waninterfaces '); + const wans = await getWanInterfacesForSite(siteId); + console.log(pretty(wans)); +} + +async function cmdHealth(args) { + const siteId = args._[1]; + if (!siteId) throw new Error('usage: health '); + const startedAt = Date.now(); + const resp = await getHealthscore(siteId); + const elapsed = Date.now() - startedAt; + if (!resp) { + bad(`getHealthscore returned null in ${elapsed}ms — check the paloalto:metrics warning above for the 400 body`); + return; + } + ok(`getHealthscore returned in ${elapsed}ms`); + console.log(pretty(resp)); +} + +async function cmdLqm(args) { + const siteId = args._[1]; + const wiCsv = args._[2]; + const metric = args.flags.metric || 'latency'; + if (!siteId || !wiCsv) throw new Error('usage: lqm [--metric latency|jitter|loss|mos]'); + if (!LQM_METRIC_NAMES[metric]) { + throw new Error(`unknown --metric "${metric}" (expected: ${Object.keys(LQM_METRIC_NAMES).join(', ')})`); + } + const wiIds = wiCsv.split(',').map((s) => s.trim()).filter(Boolean); + const startedAt = Date.now(); + const resp = await getLqmMetric(siteId, wiIds, metric); + const elapsed = Date.now() - startedAt; + if (!resp) { + bad(`getLqmMetric(${metric}) returned null in ${elapsed}ms — check the paloalto:metrics warning above for the 400 body`); + return; + } + ok(`getLqmMetric(${metric}) returned in ${elapsed}ms`); + console.log(pretty(resp)); +} + +async function cmdAlarms(args) { + const siteId = args._[1]; + const window = Number(args.flags.window) || 60; + if (!siteId) throw new Error('usage: alarms [--window ]'); + const startedAt = Date.now(); + const resp = await getAlarms(siteId, window); + const elapsed = Date.now() - startedAt; + if (!resp) { + bad(`getAlarms returned null in ${elapsed}ms — check the paloalto:metrics warning above`); + return; + } + ok(`getAlarms returned in ${elapsed}ms`); + console.log(pretty(resp)); +} + +async function cmdRaw(args) { + const method = (args._[1] || '').toUpperCase(); + const url = args._[2]; + if (!method || !url) throw new Error('usage: raw [--body \'{"json":"body"}\']'); + let body; + if (typeof args.flags.body === 'string') { + try { body = JSON.parse(args.flags.body); } + catch (e) { throw new Error(`--body is not valid JSON: ${e.message}`); } + } + const verdict = await fire({ method, url, body }); + printVerdict(verdict, args.flags); +} + +// ─── try-shapes: combinatorial schema testing ─────────────────────── + +function pickInterval5min() { return '5min'; } + +function windowIsoStart(minutes) { + return new Date(Date.now() - minutes * 60 * 1000).toISOString(); +} + +function nowIsoStart() { + return new Date().toISOString(); +} + +/** + * Build the healthscore body candidates. Each entry has: + * - `label`: short summary for the try-shapes table + * - `url`: OPTIONAL per-candidate URL override (default is + * `/sdwan/monitor/v2.0/api/monitor/aiops/health`). + * Used to test alternative endpoints (v2.1 aggregates, + * v2.6 unified metrics) in the same run. + * - `body`: request body + * + * When you land on a working shape via try-shapes, update + * `integrations/paloalto/metrics.js::getHealthscore` and add a + * regression test in `tests/paloalto.metrics.test.js`. + * + * Candidate philosophy for aiops/health v2.0 (this tenant): + * Confirmed via trip-wire on 2026-07-09: + * - `metrics` array is REJECTED at top level ("not defined") + * - `end_time` is REJECTED + * - `view` is REQUIRED (must be present) AND is a STRING ENUM + * - `filter.site`, `filter.elements` all REJECTED + * So the winning shape should be a minimal one WITHOUT `metrics` + * or `end_time`. We also try alternative endpoints in case v2.0 + * is deprecated on this tenant. + */ +function healthscoreCandidates() { + const startT = windowIsoStart(15); + const startT60 = windowIsoStart(60); + const interval = pickInterval5min(); + return [ + // ─── WINNING SHAPE (verified 2026-07-09) ─────────────────────── + // v2.6 monitor/metrics is the ONLY endpoint that returns 200 on + // the observed tenant. Kept first so it's the fast-path. + { label: 'v2.6 metrics: Healthscore metric with filter={site:[X]} ← LIVE WINNER', + url: '/sdwan/monitor/v2.6/api/monitor/metrics', + body: { + start_time: startT, interval, + metrics: [{ name: 'Healthscore', statistics: ['max'], unit: 'gauge' }], + view: {}, filter: { site: ['SITE_ID_PLACEHOLDER'] }, + } }, + // ─── v2.6 variants to probe response shape drift ─────────────── + { label: 'v2.6 metrics: statistics:["average"]', + url: '/sdwan/monitor/v2.6/api/monitor/metrics', + body: { + start_time: startT, interval, + metrics: [{ name: 'Healthscore', statistics: ['average'], unit: 'gauge' }], + view: {}, filter: { site: ['SITE_ID_PLACEHOLDER'] }, + } }, + { label: 'v2.6 metrics: view={individual:"site"}', + url: '/sdwan/monitor/v2.6/api/monitor/metrics', + body: { + start_time: startT, interval, + metrics: [{ name: 'Healthscore', statistics: ['max'], unit: 'gauge' }], + view: { individual: 'site' }, filter: { site: ['SITE_ID_PLACEHOLDER'] }, + } }, + { label: 'v2.6 metrics: view={individual:"site"} + longer window (60min)', + url: '/sdwan/monitor/v2.6/api/monitor/metrics', + body: { + start_time: startT60, interval, + metrics: [{ name: 'Healthscore', statistics: ['max'], unit: 'gauge' }], + view: { individual: 'site' }, filter: { site: ['SITE_ID_PLACEHOLDER'] }, + } }, + // ─── DEAD-END REGRESSION GUARDS (v2.0 aiops/health) ──────────── + // Kept so we notice if Prisma ever re-enables these paths. All + // currently return 400 SCHEMA_CHECK_FAIL on this tenant. + { label: 'v2.0 aiops/health: minimal {start_time, interval, view:"summary", filter:{}} (regression)', + body: { start_time: startT, interval, view: 'summary', filter: {} } }, + { label: 'v2.0 aiops/health: + end_time added back', + body: { start_time: startT, end_time: new Date().toISOString(), interval, view: 'summary', filter: {} } }, + { label: 'v2.0 aiops/health: + metrics + end_time', + body: { + start_time: startT, end_time: new Date().toISOString(), interval, + metrics: [{ name: 'Healthscore', statistics: ['max'], unit: 'gauge' }], + view: 'summary', filter: {}, + } }, + ]; +} + +/** + * Substitute placeholders (SITE_ID_PLACEHOLDER etc.) with actual + * ids before firing. Keeps the candidate list readable in code. + */ +function substitutePlaceholders(body, subs) { + const s = JSON.stringify(body); + let out = s; + for (const [ph, val] of Object.entries(subs)) { + out = out.replaceAll(`"${ph}"`, JSON.stringify(val)); + } + return JSON.parse(out); +} + +function lqmCandidates() { + // WINNING SHAPE confirmed live 2026-07-09: + // filter={ site:[X], path:[WI,...] } with view:{}, NO end_time. + // Returns metrics[].sites[].paths[].data.. + // Below variants exist so any future schema drift shows up as a + // clean try-shapes comparison rather than a silent regression. + const base = { + start_time: windowIsoStart(5), + interval: pickInterval5min(), + metrics: [{ name: 'LqmLatencyPointMetric', statistics: ['average'], unit: 'milliseconds' }], + view: {}, + }; + return [ + { label: 'filter={site:[X], path:[WI]} ← LIVE WINNER (2026-07-09)', + body: { ...base, filter: { site: ['SITE_ID_PLACEHOLDER'], path: ['WI_ID_PLACEHOLDER'] } } }, + // ─── Alternative filter-key variants (all 400 on this tenant) ── + { label: 'filter={site:[X], waninterface:[WI]} (rejected)', + body: { ...base, filter: { site: ['SITE_ID_PLACEHOLDER'], waninterface: ['WI_ID_PLACEHOLDER'] } } }, + { label: 'filter={site:[X], wan_interfaces:[WI]} (rejected)', + body: { ...base, filter: { site: ['SITE_ID_PLACEHOLDER'], wan_interfaces: ['WI_ID_PLACEHOLDER'] } } }, + { label: 'filter={site:[X]} (no circuit filter)', + body: { ...base, filter: { site: ['SITE_ID_PLACEHOLDER'] } } }, + { label: 'filter={path:[WI]} (no site filter)', + body: { ...base, filter: { path: ['WI_ID_PLACEHOLDER'] } } }, + { label: 'filter={site:[X], path:[WI]} + end_time added back (regression check)', + body: { + ...base, + end_time: new Date().toISOString(), + filter: { site: ['SITE_ID_PLACEHOLDER'], path: ['WI_ID_PLACEHOLDER'] }, + } }, + { label: 'view={individual:"path"}, filter={site:[X], path:[WI]}', + body: { ...base, view: { individual: 'path' }, filter: { site: ['SITE_ID_PLACEHOLDER'], path: ['WI_ID_PLACEHOLDER'] } } }, + ]; +} + +async function cmdTryShapes(args) { + const which = args._[1]; + const siteId = args._[2]; + if (!which || !siteId) throw new Error('usage: try-shapes []'); + + if (which === 'health') { + heading(`try-shapes health → siteId=${siteId}`); + const els = await getElementsForSite(siteId); + const elId = els?.[0]?.id; + if (!elId) info('no elements at site — element-based candidates will be skipped'); + + const subs = { SITE_ID_PLACEHOLDER: siteId }; + if (elId) subs.ELEMENT_ID_PLACEHOLDER = elId; + + const results = await runCandidates('POST', '/sdwan/monitor/v2.0/api/monitor/aiops/health', + healthscoreCandidates(), subs, args.flags); + printSummary('healthscore', results); + return; + } + + if (which === 'lqm') { + const wiCsv = args._[3]; + if (!wiCsv) throw new Error('usage: try-shapes lqm '); + heading(`try-shapes lqm → siteId=${siteId} wiIds=${wiCsv}`); + const els = await getElementsForSite(siteId); + const elId = els?.[0]?.id; + const firstWi = wiCsv.split(',')[0].trim(); + + const subs = { + SITE_ID_PLACEHOLDER: siteId, + WI_ID_PLACEHOLDER: firstWi, + }; + if (elId) subs.ELEMENT_ID_PLACEHOLDER = elId; + + const results = await runCandidates('POST', '/sdwan/monitor/v2.0/api/monitor/lqm_point_metrics', + lqmCandidates(), subs, args.flags); + printSummary('lqm_point_metrics', results); + return; + } + + // Metric name + unit matrix probe. Fires each {name, unit} combo + // and reports which return 200. Use when a specific metric key + // (loss, mos, etc.) is 400-ing with METRIC_UNIT_NOT_SUPPORTED + // or METRIC_NOT_FOUND. Assumes filter + view shape is already + // solved (uses the current winning filter={site:[X], path:[WI]}). + const LQM_MATRIX_TARGETS = { + 'lqm-loss': [ + // Verified winner (2026-07-09): LqmPktLossPointMetric + percentage. + { name: 'LqmPktLossPointMetric', unit: 'percentage' }, // ← LIVE WINNER + { name: 'LqmPktLossPointMetric', unit: 'percent' }, + { name: 'LqmPktLossPointMetric', unit: 'pct' }, + { name: 'LqmPktLossPointMetric', unit: 'ratio' }, + { name: 'LqmPktLossPointMetric', unit: 'count' }, + { name: 'LqmPktLossPointMetric', unit: 'gauge' }, + { name: 'LqmPktLossPointMetric', unit: 'Percentage' }, // regression check — was wrong pre-fix + { name: 'LqmPacketLossPointMetric', unit: 'percentage' }, + { name: 'LqmLossPointMetric', unit: 'percentage' }, + { name: 'LqmPacketDropPointMetric', unit: 'percentage' }, + { name: 'LqmPktLossPercentPointMetric', unit: 'percentage' }, + { name: 'LqmPktLossPctPointMetric', unit: 'percentage' }, + ], + 'lqm-mos': [ + // Verified winner (2026-07-09): LqmMosPointMetric + count. + { name: 'LqmMosPointMetric', unit: 'count' }, // ← LIVE WINNER + { name: 'LqmMosPointMetric', unit: 'score' }, + { name: 'LqmMosPointMetric', unit: 'mos' }, + { name: 'LqmMosPointMetric', unit: 'ratio' }, + { name: 'LqmMosPointMetric', unit: 'gauge' }, + { name: 'LqmMosScorePointMetric', unit: 'count' }, + { name: 'LqmMeanOpinionScorePointMetric', unit: 'count' }, + ], + 'lqm-latency': [ + // Verified working (2026-07-09) via `lqm --metric latency`. + { name: 'LqmLatencyPointMetric', unit: 'milliseconds' }, // ← LIVE WINNER + { name: 'LqmLatencyPointMetric', unit: 'ms' }, + { name: 'LqmLatencyPointMetric', unit: 'Milliseconds' }, + { name: 'LqmLatencyPointMetric', unit: 'count' }, + { name: 'LqmRttLatencyPointMetric', unit: 'milliseconds' }, + { name: 'LqmLatencyRttPointMetric', unit: 'milliseconds' }, + ], + 'lqm-jitter': [ + // Currently working via fallback scanner; run this to confirm + // the "true" name/unit and whether jitter is directional. + { name: 'LqmJitterPointMetric', unit: 'milliseconds' }, // ← current default + { name: 'LqmJitterPointMetric', unit: 'ms' }, + { name: 'LqmJitterPointMetric', unit: 'count' }, + { name: 'LqmRttJitterPointMetric', unit: 'milliseconds' }, + { name: 'LqmJitterMsPointMetric', unit: 'milliseconds' }, + ], + }; + if (LQM_MATRIX_TARGETS[which]) { + const wiCsv = args._[3]; + if (!wiCsv) throw new Error(`usage: try-shapes ${which} `); + heading(`try-shapes ${which} → siteId=${siteId} wiIds=${wiCsv}`); + const wiIds = wiCsv.split(',').map((s) => s.trim()).filter(Boolean); + const nameUnitMatrix = LQM_MATRIX_TARGETS[which]; + + const candidates = nameUnitMatrix.map((mu) => ({ + label: `name="${mu.name}", unit="${mu.unit}"`, + body: { + start_time: windowIsoStart(5), + interval: pickInterval5min(), + metrics: [{ name: mu.name, statistics: ['average'], unit: mu.unit }], + view: {}, + filter: { site: [siteId], path: wiIds }, + }, + })); + + const results = await runCandidates( + 'POST', + '/sdwan/monitor/v2.0/api/monitor/lqm_point_metrics', + candidates, {}, args.flags, + ); + printSummary(`${which} name+unit matrix`, results); + return; + } + + throw new Error(`unknown try-shapes target "${which}" — expected "health", "lqm", "lqm-latency", "lqm-jitter", "lqm-loss", or "lqm-mos"`); +} + +/** + * Run a list of candidate bodies against a default URL (or the + * candidate's own `url` override if provided). Each candidate is + * fired serially so trip-wire ordering is deterministic. Skips + * candidates that reference a placeholder we don't have (e.g. + * ELEMENT_ID_PLACEHOLDER when the site has no elements). + */ +async function runCandidates(method, defaultUrl, candidates, subs, flags) { + const results = []; + for (let i = 0; i < candidates.length; i += 1) { + const c = candidates[i]; + // Skip candidates that rely on a placeholder we don't have. + const needsEl = JSON.stringify(c.body).includes('ELEMENT_ID_PLACEHOLDER'); + if (needsEl && !subs.ELEMENT_ID_PLACEHOLDER) { + results.push({ ...c, status: 'skip', reason: 'no element id available' }); + console.log(` [${i + 1}/${candidates.length}] ${c.label} → ⚠️ skipped (no element id)`); + continue; + } + const body = substitutePlaceholders(c.body, subs); + const targetUrl = c.url || defaultUrl; + process.stdout.write(` [${i + 1}/${candidates.length}] ${c.label} → ${ICON_WAIT}`); + const verdict = await fire({ method, url: targetUrl, body }); + if (verdict.ok) { + const shape = shapeSummary(verdict.response); + console.log(`${ICON_OK} 200 (${verdict.elapsedMs}ms) ${shape}`); + results.push({ ...c, verdict, shape }); + } else { + const schemaMsg = extractSchemaError(verdict.response) || `HTTP ${verdict.status}`; + console.log(`${ICON_ERR} ${verdict.status || 'network'} (${verdict.elapsedMs}ms) ${schemaMsg}`); + results.push({ ...c, verdict, schemaMsg }); + } + if (flags.showBody) { + console.log(` url: ${targetUrl}`); + console.log(` req: ${shortJson(body, 200)}`); + } + } + return results; +} + +/** + * One-line summary of a 200 response for the try-shapes table. + * Highlights whether the metric series has actual data points and + * which keys are present under `series[0].view` — those are the + * two things you always care about when comparing shapes. + */ +function shapeSummary(resp) { + if (!resp || typeof resp !== 'object') return '(non-object response)'; + const metric = resp?.metrics?.[0]; + if (!metric) return `top-level=[${Object.keys(resp).join(',')}]`; + + // Preferred (live) shape: metrics[0].sites[0].{paths[] | healthscore | data.} + if (Array.isArray(metric.sites) && metric.sites.length > 0) { + const s0 = metric.sites[0]; + if (Array.isArray(s0.paths)) { + // LQM shape: paths[].data. + const p0 = s0.paths[0]; + const dataKeys = p0?.data ? Object.keys(p0.data).filter((k) => k !== 'sample_completeness').join(',') : '(none)'; + const firstNumeric = p0?.data ? Object.entries(p0.data).find(([k, v]) => k !== 'sample_completeness' && typeof v === 'number') : null; + const lastVal = firstNumeric ? `${firstNumeric[0]}=${firstNumeric[1]}` : '(no data)'; + return `sites=${metric.sites.length} paths=${s0.paths.length} data.keys=[${dataKeys}] first=${lastVal}`; + } + // Healthscore v2.6 shape: sites[].healthscore or sites[].data.score + const siteKeys = Object.keys(s0).join(','); + const scoreKey = ['healthscore', 'health_score', 'score', 'value'].find((k) => typeof s0[k] === 'number'); + const nestedScoreKey = s0.data ? ['healthscore', 'health_score', 'score', 'value'].find((k) => typeof s0.data[k] === 'number') : null; + let val = '(none)'; + if (scoreKey) val = `${scoreKey}=${s0[scoreKey]}`; + else if (nestedScoreKey) val = `data.${nestedScoreKey}=${s0.data[nestedScoreKey]}`; + return `sites=${metric.sites.length} site[0].keys=[${siteKeys}] score=${val}`; + } + + // Legacy (pan.dev) shape: metrics[0].series[].data[].value + const series = metric.series || []; + const s0 = series[0]; + const viewKeys = s0?.view ? Object.keys(s0.view).join(',') : '(none)'; + const dataLen = Array.isArray(s0?.data) ? s0.data.length : 0; + const lastVal = dataLen > 0 ? s0.data[dataLen - 1]?.value : '(no data)'; + return `series=${series.length} view.keys=[${viewKeys}] data=${dataLen}pt lastVal=${lastVal}`; +} + +function printSummary(label, results) { + heading(`Summary: ${label}`); + const winners = results.filter((r) => r.verdict?.ok); + const losers = results.filter((r) => r.verdict && !r.verdict.ok); + const skips = results.filter((r) => r.status === 'skip'); + + const urlSuffix = (r) => r.url ? ` [url override: ${r.url}]` : ''; + + if (winners.length === 0) { + bad(`no candidate passed schema check`); + } else { + ok(`${winners.length} candidate(s) passed:`); + for (const w of winners) { + console.log(` • ${w.label} → ${w.shape}${urlSuffix(w)}`); + } + } + if (losers.length > 0) { + console.log(`\n ${losers.length} candidate(s) rejected:`); + for (const l of losers) { + console.log(` • ${l.label} → ${l.schemaMsg}${urlSuffix(l)}`); + } + } + if (skips.length > 0) { + console.log(`\n ${skips.length} skipped:`); + for (const s of skips) console.log(` • ${s.label} → ${s.reason}${urlSuffix(s)}`); + } + console.log(''); + info(`When you find a winner, update integrations/paloalto/metrics.js`); + info(`and add a regression test in tests/paloalto.metrics.test.js.`); + info(`Re-run this command with --show-body to see the exact request bodies.`); +} + +// ─── Main ─────────────────────────────────────────────────────────── + +function printHelp() { + // Extract the usage docblock from the top of this file so help stays + // in sync with the docstring. Falls back to a short summary if the + // file isn't readable (e.g. bundled). + console.log([ + 'Prisma SD-WAN API probe', + '', + 'Usage:', + ' node scripts/prismaProbe.js [args] [flags]', + '', + 'Subcommands:', + ' discover Full store discovery', + ' site Resolve store → site', + ' elements List site elements', + ' waninterfaces List site waninterfaces', + ' health Fetch healthscore', + ' lqm [--metric X] Fetch LQM metric (latency|jitter|loss|mos)', + ' alarms [--window minutes] Fetch alarms', + ' raw [--body JSON] Arbitrary authenticated request', + ' try-shapes health Test N healthscore body shapes', + ' try-shapes lqm Test N LQM body shapes', + ' try-shapes lqm-latency Sweep {name × unit} combos for latency', + ' try-shapes lqm-jitter Sweep {name × unit} combos for jitter', + ' try-shapes lqm-loss Sweep {name × unit} combos for packet loss', + ' try-shapes lqm-mos Sweep {name × unit} combos for MOS', + '', + 'Global flags:', + ' --json JSON output', + ' --show-body Show request body on success too', + ' --quiet Suppress axios request log line', + ' --help, -h This help', + '', + 'Examples:', + ' node scripts/prismaProbe.js discover 782', + ' node scripts/prismaProbe.js try-shapes health 16158173173100144', + ' node scripts/prismaProbe.js lqm 16158173173100144 16158173176610209 --metric loss', + ' node scripts/prismaProbe.js try-shapes lqm-loss 16158173173100144 16158173176610209,1666974885552003096', + ' node scripts/prismaProbe.js raw POST /sdwan/v3.7/api/events/query --body \'{"limit":{"count":5}}\'', + ].join('\n')); +} + +async function main() { + const args = parseArgs(process.argv); + if (args.flags.help || args._.length === 0) { + printHelp(); + process.exit(args.flags.help ? 0 : 1); + return; + } + + const sub = args._[0]; + const dispatch = { + discover: cmdDiscover, + site: cmdSite, + elements: cmdElements, + waninterfaces: cmdWaninterfaces, + health: cmdHealth, + lqm: cmdLqm, + alarms: cmdAlarms, + raw: cmdRaw, + 'try-shapes': cmdTryShapes, + }; + const fn = dispatch[sub]; + if (!fn) { + bad(`unknown subcommand "${sub}"`); + printHelp(); + process.exit(1); + return; + } + + try { + await fn(args); + } catch (err) { + bad(err.message); + process.exit(1); + } +} + +main(); diff --git a/services/enrichment/sdwanEnrichment.js b/services/enrichment/sdwanEnrichment.js new file mode 100644 index 0000000..a8bfefc --- /dev/null +++ b/services/enrichment/sdwanEnrichment.js @@ -0,0 +1,679 @@ +// src/services/enrichment/sdwanEnrichment.js +// +// One-stop composer for Prisma SD-WAN data used by /phonestatus's +// WAN follow-up and /voicediag's WAN check bucket. Same "fire the +// fetches in parallel + normalise into a stable shape + preserve +// per-metric failures in errors[]" pattern used by the Meraki +// enrichment layer. +// +// Contract (what `collectSdwanForStore(storeNum)` returns): +// +// { +// storeNum, +// site: { id, name, storeNum, description } | null, +// elements: [{ id, name, model, serial_number, connected }], +// healthscore: { value: 0-100, breakdown: {...} } | null, +// links: [ +// { +// interfaceId, // Prisma waninterface id +// interfaceName, // human label +// elementId, // null in phase 1 — waninterfaces are site-scoped, not element-scoped +// transportType, // "primary" | "secondary" | "backup" | ... (from used_for) +// up: boolean | null, // waninterface admin_up (proxy until runtime status is wired) +// latencyMs: number | null, +// jitterMs: number | null, +// lossPct: number | null, +// mos: number | null, +// }, +// ... +// ], +// alarms: { +// last1h: { critical: N, major: N, minor: N }, +// samples: [{ code, message, severity, ts }] // top 5, most recent +// }, +// errors: [{ scope, message }], // per-metric failure lines +// fetchedAt, +// } +// +// Design rules: +// - Never throws. A caller must be able to render "we tried, here's +// what we got, here's what failed" without a top-level try/catch. +// - `site: null` short-circuits everything else — the composer +// returns immediately with `errors: []` and empty collections. +// This is the "not a Prisma-managed store" happy path. +// - Each metric fetch is independent (`Promise.allSettled`). A +// failure adds an entry to `errors[]` and leaves the +// corresponding field null / empty; it does NOT propagate. +// - Response-shape assumptions are minimal and defensive: we +// read `res.metrics || res.data || res.items || []` and pick +// out the fields we recognise, tolerating extras/missing. +// - No caching here — each caller gets a fresh pull. Caching +// lives one layer down (sites + elements are cached; live +// metrics are always fresh). + +import { logger } from '../../utils/logger.js'; +import { + findSdwanSiteForStore, + getElementsForSite, + getWanInterfacesForSite, +} from '../../integrations/paloalto/sites.js'; +import { + getHealthscore, + getLqmMetric, + getAlarms, +} from '../../integrations/paloalto/metrics.js'; + +/** + * Compose everything /voicediag + /phonestatus care about for a + * store's Prisma SD-WAN posture. Never throws — always returns the + * shape documented at the top of this file. + * + * @param {string|number} storeNum + * @param {object} [opts] + * @param {number} [opts.windowMinutes] Look-back window for healthscore + LQM. + * Also used as the alarm window unless `alarmWindowMinutes` is set. + * Defaults to `WAN_STANDARD_WINDOW_MINUTES` env var or 15. + * @param {number} [opts.alarmWindowMinutes] Override the alarm window + * independently (defaults to `max(60, windowMinutes)` since alarms + * at < 1h are usually too noisy to be actionable). + * @returns {Promise} See file header for shape. Includes + * `window` reporting the effective look-back so callers/renderers + * can surface it. + */ +export async function collectSdwanForStore(storeNum, opts = {}) { + const startedAt = Date.now(); + const emptyLinks = []; + const errors = []; + + const windowMinutes = normalizeWindowMinutes(opts.windowMinutes); + const alarmWindowMinutes = normalizeWindowMinutes( + opts.alarmWindowMinutes ?? Math.max(60, windowMinutes), + ); + + const site = await findSdwanSiteForStore(storeNum).catch((err) => { + errors.push({ scope: 'sites', message: err.message }); + return null; + }); + + if (!site) { + logger('sdwan:enrich', `No Prisma site for store ${storeNum} (not a Prisma-managed site)`, 'debug'); + return { + storeNum: String(storeNum), + site: null, + elements: [], + healthscore: null, + links: emptyLinks, + alarms: emptyAlarms(), + errors, + fetchedAt: new Date().toISOString(), + window: { minutes: windowMinutes, alarmMinutes: alarmWindowMinutes }, + }; + } + + // Fetch config (elements + waninterfaces) in parallel — both are + // 4h-cached at the integration layer so this is a hot path on + // steady state. + const [elementsRes, wanInterfacesRes] = await Promise.allSettled([ + getElementsForSite(site.id), + getWanInterfacesForSite(site.id), + ]); + + recordFailure(errors, 'elements', elementsRes); + recordFailure(errors, 'waninterfaces', wanInterfacesRes); + + const elements = valueOf(elementsRes) || []; + const wanInterfaces = valueOf(wanInterfacesRes) || []; + const waninterfaceIds = wanInterfaces.map((w) => w.id).filter(Boolean); + + // Metric-endpoint dependency matrix (verified against a live SASE tenant): + // getHealthscore → site only (filter shape is empty; site match done locally) + // getLqmMetric → requires waninterfaceIds (per-path) + // getAlarms → site only + const canFetchLqm = waninterfaceIds.length > 0; + const [ + healthscoreRes, + latencyRes, + jitterRes, + lossRes, + mosRes, + alarmsRes, + ] = await Promise.allSettled([ + getHealthscore(site.id, windowMinutes), + canFetchLqm ? getLqmMetric(site.id, waninterfaceIds, 'latency', windowMinutes) : Promise.resolve(null), + canFetchLqm ? getLqmMetric(site.id, waninterfaceIds, 'jitter', windowMinutes) : Promise.resolve(null), + canFetchLqm ? getLqmMetric(site.id, waninterfaceIds, 'loss', windowMinutes) : Promise.resolve(null), + canFetchLqm ? getLqmMetric(site.id, waninterfaceIds, 'mos', windowMinutes) : Promise.resolve(null), + getAlarms(site.id, alarmWindowMinutes), + ]); + + recordFailure(errors, 'healthscore', healthscoreRes); + recordFailure(errors, 'lqm.latency', latencyRes); + recordFailure(errors, 'lqm.jitter', jitterRes); + recordFailure(errors, 'lqm.loss', lossRes); + recordFailure(errors, 'lqm.mos', mosRes); + recordFailure(errors, 'alarms', alarmsRes); + + // Healthscore response covers the whole tenant (see filter shape + // note in metrics.js). Match the requested site locally. + const healthscore = parseHealthscore(valueOf(healthscoreRes), site.id); + const links = buildLinkRows({ + wanInterfaces, + metricResponses: { + latency: valueOf(latencyRes), + jitter: valueOf(jitterRes), + loss: valueOf(lossRes), + mos: valueOf(mosRes), + }, + }); + // Second-line defense: if events/query silently ignored our + // query.site filter, parseAlarms filters locally by site_id. + const alarms = parseAlarms(valueOf(alarmsRes), site.id); + + // Debug shape probes: on high error counts these give us the + // exact top-level keys Prisma returned, which is the fastest way + // to spot schema drift. Kept at `warn` level while the response + // shape is still under active reverse-engineering. + const hsRaw = valueOf(healthscoreRes); + if (hsRaw && healthscore?.value == null) { + // v2.6 metrics response is expected to shape as: + // metrics[0].sites[].healthscore (or .data.score, etc.) + const m0 = hsRaw.metrics?.[0]; + const s0 = m0?.sites?.[0]; + logger('sdwan:enrich', + `healthscore returned but parseHealthscore extracted no value — ` + + `top-level keys: [${Object.keys(hsRaw).join(',')}], ` + + `metrics[0] keys: [${m0 ? Object.keys(m0).join(',') : 'n/a'}], ` + + `sites count: ${m0?.sites?.length ?? 'n/a'}, ` + + `sites[0] keys: ${s0 ? '[' + Object.keys(s0).join(',') + ']' : 'n/a'}, ` + + `sites[0] preview: ${JSON.stringify(s0 || null).slice(0, 200)}`, + 'warn'); + } + const linksWithSamples = links.filter((l) => l.latencyMs != null || l.jitterMs != null || l.lossPct != null || l.mos != null); + if (waninterfaceIds.length > 0 && linksWithSamples.length === 0) { + const anyLqmRaw = valueOf(latencyRes) || valueOf(jitterRes) || valueOf(lossRes) || valueOf(mosRes); + if (anyLqmRaw) { + // Live shape: metrics[0].sites[0].paths[N].data. + const m0 = anyLqmRaw.metrics?.[0]; + const s0 = m0?.sites?.[0]; + const p0 = s0?.paths?.[0]; + logger('sdwan:enrich', + `LQM responses returned but 0 links have samples — ` + + `top-level keys: [${Object.keys(anyLqmRaw).join(',')}], ` + + `metrics[0] keys: [${m0 ? Object.keys(m0).join(',') : 'n/a'}], ` + + `sites count: ${m0?.sites?.length ?? 'n/a'}, ` + + `paths[0]: ${JSON.stringify(p0 || null).slice(0, 200)}`, + 'warn'); + } + } + + const elapsed = Date.now() - startedAt; + const alarmDiag = alarms?._diag + ? ` alarms=${alarms._diag.rawEventCount}raw→${alarms._diag.rawEventCount - alarms._diag.droppedForSiteMismatch}scoped` + : ''; + logger( + 'sdwan:enrich', + `store ${storeNum} → site ${site.name} (${site.id}): ${elements.length} elements, ` + + `${links.length} links, health=${healthscore?.value ?? 'n/a'},${alarmDiag} errors=${errors.length}, ` + + `window=${windowMinutes}m/alarms=${alarmWindowMinutes}m, ${elapsed}ms`, + ); + + return { + storeNum: String(storeNum), + site: { + id: site.id, + name: site.name, + storeNum: String(storeNum), + description: site.description || '', + }, + elements: elements.map((e) => ({ + id: e.id, + name: e.name, + model: e.model, + serial_number: e.serial_number, + connected: e.connected, + })), + healthscore, + links, + alarms, + errors, + fetchedAt: new Date().toISOString(), + // Report the effective look-back so renderers can surface it and + // callers can verify the request they intended was honored. + window: { minutes: windowMinutes, alarmMinutes: alarmWindowMinutes }, + }; +} + +/** + * Snap an arbitrary window request to a reasonable, sane value. + * Falls back to the env default (WAN_STANDARD_WINDOW_MINUTES) or 15. + * Prisma's own upper bound seems to sit around 30 days for aiops + * queries, but voice-quality signal degrades past a day of averaging + * so we cap at 24h for now. + */ +function normalizeWindowMinutes(m) { + const raw = Number.isFinite(Number(m)) && Number(m) > 0 + ? Number(m) + : Number(process.env.WAN_STANDARD_WINDOW_MINUTES) || 15; + // Clamp to [1, 1440] (1 min .. 24 h). + return Math.max(1, Math.min(1440, Math.floor(raw))); +} + +// ─── Response normalisers ───────────────────────────────────────── + +function emptyAlarms() { + return { last1h: { critical: 0, major: 0, minor: 0 }, samples: [] }; +} + +function valueOf(settledResult) { + if (!settledResult) return null; + return settledResult.status === 'fulfilled' ? settledResult.value : null; +} + +function recordFailure(errors, scope, settledResult) { + if (settledResult?.status === 'rejected') { + errors.push({ scope, message: settledResult.reason?.message || String(settledResult.reason) }); + } +} + +/** + * Prisma's aiops/health response carries at least one + * `metrics[]` array with per-time-bucket values. We collapse to + * "the most recent site-level score" plus an optional breakdown + * of contributing sub-scores when present. Defensive on shape: + * an unrecognised payload just returns null. + */ +// The three parse* helpers below are exported for direct unit +// testing (tests/sdwanEnrichment.test.js). They're not part of the +// public composer surface — callers should use `collectSdwanForStore`. +/** + * Extract the last healthscore value from a v2.6 monitor/metrics + * response. Verified live on this tenant on 2026-07-09: + * + * metrics[0].series[0].data[0].datapoints[N].value + * + * The `data[0]` is a WRAPPER (`{statistics: 'max', datapoints: [...]}`) + * whose `datapoints` array holds the actual time-series. We take + * the most recent datapoint. That's what "healthscore right now" + * means from a user's perspective. + * + * Response is already server-side scoped to our site (via + * filter.site request), so we don't need to filter by siteId here — + * but we keep the parameter for backwards-compat with the other + * shapes we still tolerate as fallbacks. + * + * Shapes tolerated (in order of preference): + * 1. v2.6 metrics wrapper: metrics[].series[].data[].datapoints[].value ← LIVE WINNER + * 2. v2.6 per-site rollup: metrics[].sites[].{healthscore | score | data.score} + * 3. Legacy pan.dev: metrics[].series[].data[].value + */ +export function parseHealthscore(resp, siteId = null) { + if (!resp || typeof resp !== 'object') return null; + const metrics = resp.metrics || resp.data || []; + if (!Array.isArray(metrics) || metrics.length === 0) return null; + + // ── Shape 1 (LIVE): metrics[].series[].data[].datapoints[].value ── + for (const m of metrics) { + const series0 = Array.isArray(m?.series) ? m.series[0] : null; + if (!series0) continue; + const dataWrapper0 = Array.isArray(series0.data) ? series0.data[0] : null; + if (!dataWrapper0 || !Array.isArray(dataWrapper0.datapoints)) continue; + const lastDp = [...dataWrapper0.datapoints].reverse().find( + (p) => Number.isFinite(Number(p?.value)), + ); + if (lastDp) { + return { value: Math.round(Number(lastDp.value)), breakdown: {} }; + } + } + + // ── Shape 2 (defensive): metrics[].sites[].healthscore / .score ── + for (const m of metrics) { + if (!/health/i.test(m?.name || '') && m !== metrics[0]) continue; + const sites = m?.sites; + if (Array.isArray(sites)) { + const match = sites.find((s) => { + const sid = s?.site_id || s?.id || s?.site; + return sid && String(sid) === String(siteId); + }) || sites[0]; + const v = extractHealthscoreValue(match); + if (v != null) return { value: Math.round(v), breakdown: {} }; + } + } + + // ── Shape 3 (legacy pan.dev): metrics[].series[].data[].value ── + const primary = metrics.find((m) => /health/i.test(m?.name || '')) || metrics[0]; + const allSeries = primary?.series || primary?.view?.series || []; + let candidates = Array.isArray(allSeries) ? allSeries : []; + if (siteId) { + const scoped = candidates.filter((s) => { + const v = s?.view?.site || s?.view?.site_id || s?.site_id; + return v && String(v) === String(siteId); + }); + if (scoped.length > 0) candidates = scoped; + } + const points = Array.isArray(candidates[0]?.data) + ? candidates[0].data + : (Array.isArray(candidates) ? candidates : []); + const lastPoint = [...points].reverse().find((p) => Number.isFinite(Number(p?.value))); + if (lastPoint) { + const value = Math.round(Number(lastPoint.value)); + const breakdown = {}; + for (const m of metrics) { + if (m === primary) continue; + const s = m?.series || m?.view?.series || []; + const pts = Array.isArray(s[0]?.data) ? s[0].data : (Array.isArray(s) ? s : []); + const last = [...pts].reverse().find((p) => Number.isFinite(Number(p?.value))); + if (last && m?.name) { + breakdown[m.name] = Math.round(Number(last.value)); + } + } + return { value, breakdown }; + } + + return null; +} + +/** + * Extract a healthscore value from an unknown-shape object by + * looking for a numeric field with a healthscore-like key. Handles + * both the top-level case (obj.score) and one-level-of-nesting + * (obj.data.score). Falls through to the first plausible numeric + * field within the 0-100 range if no known key matches. + */ +function extractHealthscoreValue(obj) { + if (!obj || typeof obj !== 'object') return null; + const knownKeys = ['healthscore', 'health_score', 'score', 'value', 'health']; + const candidates = [obj, obj.data, obj.metrics, obj.summary].filter(Boolean); + for (const cand of candidates) { + for (const k of knownKeys) { + const v = cand[k]; + if (typeof v === 'number' && Number.isFinite(v)) return v; + } + } + // Last-resort: first 0-100 numeric field. + for (const cand of candidates) { + for (const [k, v] of Object.entries(cand)) { + if (typeof v !== 'number' || !Number.isFinite(v)) continue; + if (k === 'sample_completeness') continue; + if (v >= 0 && v <= 100) return v; + } + } + return null; +} + +/** +/** + * Per-metric field-name lookup inside the `data` object of the + * observed live response shape: + * + * metrics[].sites[].paths[].data = { + * sample_completeness: 100, + * rtt_latency: 22.0121 // ← metric-specific value key + * } + * + * Loss and MOS are directional — Prisma returns them as separate + * uplink/downlink keys. Voice quality degrades with loss in EITHER + * direction and MOS reflects the WORSE endpoint's experience, so + * DIRECTIONAL_LQM_KEYS below folds each pair via the appropriate + * combiner (max for loss, min for MOS since low-MOS is worse). + * + * Live shapes verified 2026-07-09 via prismaProbe try-shapes: + * latency → data.rtt_latency (RTT scalar) + * jitter → data.rtt_jitter (assumed via fallback) (RTT scalar) + * loss → data.{downlink,uplink}_pkt_loss_avg (directional) + * mos → data.{downlink,uplink}_mos_{avg,min,max} (directional × 3 stats) + * + * The `_fallback` scan below catches drift by picking the first + * numeric non-completeness key in the data object. + */ +const LQM_DATA_KEYS = { + latency: ['rtt_latency', 'latency', 'latency_ms'], + jitter: ['rtt_jitter', 'jitter', 'jitter_ms', 'one_way_jitter'], + // Loss & MOS: direct-hit keys only. Directional handling folds + // uplink/downlink pairs BEFORE this list is consulted. + loss: ['pkt_loss_pct', 'packet_loss', 'pkt_loss', 'loss', 'loss_pct'], + mos: ['mos', 'mos_score', 'mean_opinion_score'], +}; + +// Directional metric keys — Prisma reports these as two separate +// numbers (uplink/downlink). extractLqmValue folds them into a +// single value per the `combine` strategy: worst-case in each +// metric's own semantic (max for loss/latency-like where higher is +// worse; min for MOS where lower is worse). +const DIRECTIONAL_LQM_KEYS = { + loss: { + keys: ['downlink_pkt_loss_avg', 'uplink_pkt_loss_avg'], + combine: Math.max, // loss anywhere hurts voice quality + }, + mos: { + // Average per direction. `_min` and `_max` variants also exist in + // the response (e.g. downlink_mos_min) — we pick avg because + // it's the typical experience over the window, not a transient + // dip. If we picked _min we'd get spuriously bad readings from + // single-sample glitches. + keys: ['downlink_mos_avg', 'uplink_mos_avg'], + combine: Math.min, // low MOS = worse audio, so take the worse direction + }, +}; + +function extractLqmValue(dataObj, metricKey) { + if (!dataObj || typeof dataObj !== 'object') return null; + + // Directional handling FIRST — if a metric has known directional + // split keys, fold them (e.g. loss = max(downlink, uplink)) so + // asymmetric loss patterns don't silently disappear when we pick + // whichever direction was 0. + const dir = DIRECTIONAL_LQM_KEYS[metricKey]; + if (dir) { + const nums = dir.keys + .map((k) => dataObj[k]) + .filter((v) => typeof v === 'number' && Number.isFinite(v)); + if (nums.length > 0) return dir.combine(...nums); + } + + const knownKeys = LQM_DATA_KEYS[metricKey] || []; + for (const k of knownKeys) { + if (typeof dataObj[k] === 'number' && Number.isFinite(dataObj[k])) { + return dataObj[k]; + } + } + // Defensive fallback: first numeric key that isn't the + // sample-quality marker. This catches Prisma renaming a key + // between tenant versions without silently going to null. + for (const [k, v] of Object.entries(dataObj)) { + if (k === 'sample_completeness') continue; + if (typeof v === 'number' && Number.isFinite(v)) return v; + } + return null; +} + +/** + * Build per-path (per-waninterface) rows from the site's waninterface + * config + the four LQM metric responses. + * + * The waninterface config is the SOURCE OF TRUTH for the link list — + * every configured WAN interface at the site becomes a row here, even + * if Prisma has no recent LQM samples for it. LQM metric responses + * are layered in as overlays: last non-null sample per interface for + * each of the four metrics. Missing samples stay `null`. + * + * Prisma's LQM response shape on this tenant (verified live via + * /scripts/prismaProbe.js on 2026-07-09) — NOT the pan.dev shape: + * + * { + * metrics: [{ + * name: 'LqmLatencyPointMetric', + * unit: 'milliseconds', + * sites: [{ + * site_id: '', + * paths: [{ + * path_id: '', + * remote_site_id: '0', + * data: { + * sample_completeness: 100, + * rtt_latency: 22.0121 // ← key varies by metric + * } + * }, ...] + * }] + * }] + * } + * + * Note: `data` is a SINGLE OBJECT, NOT an array of time-series + * points. Whatever `interval` you request, this shape gives you + * one point per path (the aggregate for the window). The pan.dev + * docs describing series[].data[].value are for a different + * response variant. + * + * `up` is set from waninterface config's `admin_up` (the closest + * available proxy without a per-interface runtime status call). A + * more accurate "is this circuit currently carrying traffic" signal + * would need the /waninterfaces/{id}/status endpoint per element, + * which is deferred to a future phase. + */ +export function buildLinkRows({ wanInterfaces, metricResponses }) { + const rows = new Map(); + for (const w of Array.isArray(wanInterfaces) ? wanInterfaces : []) { + rows.set(w.id, { + interfaceId: w.id, + interfaceName: w.name || w.id, + elementId: null, + // usedFor is 'primary' / 'secondary' / 'lte-backup' etc. — + // stand-in transport label until wan_networks config is wired + // in phase 2 for the real MPLS/BROADBAND/LTE labels. + transportType: w.usedFor || null, + up: w.adminUp, // admin state = closest available "is it up" until we wire runtime status + latencyMs: null, + jitterMs: null, + lossPct: null, + mos: null, + // Diagnostic side-channel — surfaced in details so the + // operator can see whether Prisma's sample was fresh (100) + // or partial (e.g. 20 = only 1 of 5 minute-buckets had data). + sampleCompleteness: null, + }); + } + + const stampFromResponse = (resp, metricKey, apply) => { + if (!resp?.metrics) return; + for (const metric of resp.metrics) { + // Preferred shape (verified live on this tenant): + // metrics[].sites[].paths[].data. + if (Array.isArray(metric?.sites)) { + for (const site of metric.sites) { + for (const p of site?.paths || []) { + const pathId = p?.path_id; + if (!pathId || !rows.has(pathId)) continue; + const value = extractLqmValue(p?.data, metricKey); + if (value != null) apply(rows.get(pathId), value); + if (typeof p?.data?.sample_completeness === 'number') { + rows.get(pathId).sampleCompleteness = p.data.sample_completeness; + } + } + } + continue; + } + // Fallback shape (pan.dev-documented, may exist on other + // tenants or API versions): metrics[].series[].data[].value + // keyed by view.path / view.waninterface. + const series = metric?.series || metric?.view?.series || []; + for (const s of series) { + const wiId = s?.view?.path + || s?.view?.waninterface + || s?.view?.wan_interface_id + || s?.view?.interface_id; + if (!wiId || !rows.has(wiId)) continue; + const points = Array.isArray(s?.data) ? s.data : []; + const last = [...points].reverse().find((p) => p?.value !== null && p?.value !== undefined); + if (!last) continue; + apply(rows.get(wiId), Number(last.value)); + } + } + }; + + stampFromResponse(metricResponses?.latency, 'latency', (row, v) => { row.latencyMs = round(v, 1); }); + stampFromResponse(metricResponses?.jitter, 'jitter', (row, v) => { row.jitterMs = round(v, 1); }); + stampFromResponse(metricResponses?.loss, 'loss', (row, v) => { row.lossPct = round(v, 2); }); + stampFromResponse(metricResponses?.mos, 'mos', (row, v) => { row.mos = round(v, 2); }); + + // Derive up-state when admin_up is not exposed by the waninterface + // config API. If we have ANY LQM sample for a path, the path is + // provably carrying traffic — that's a stronger signal than admin + // state anyway. We only upgrade unknown (null) rows here; explicit + // adminUp=false (admin-disabled) stays down. + for (const row of rows.values()) { + if (row.up === null || row.up === undefined) { + const hasSample = row.latencyMs != null + || row.jitterMs != null + || row.lossPct != null + || row.mos != null; + if (hasSample) row.up = true; + } + } + + return [...rows.values()]; +} + +export function parseAlarms(resp, siteId = null) { + if (!resp || typeof resp !== 'object') return emptyAlarms(); + const items = resp.items || resp.data || resp.alarms || []; + if (!Array.isArray(items)) return emptyAlarms(); + + const counts = { critical: 0, major: 0, minor: 0 }; + const samples = []; + let droppedForSiteMismatch = 0; + for (const a of items) { + // Events/query returns both open AND cleared alarms in the same + // response — filter out already-resolved ones so "N alarms in + // the last hour" reflects still-active issues. + if (a?.cleared === true) continue; + + // Defensive site scoping: if the tenant's events/query silently + // ignores our `query.site` filter (schema quirk we hit before), + // we'd otherwise report tenant-wide alarms as this-site alarms + // — a badly misleading number. Match locally by site_id when a + // siteId is provided AND the event carries one. Events with NO + // site_id are kept as-is (e.g. tenant-scoped events that legitimately + // don't belong to a specific site). + if (siteId) { + const eventSiteId = a?.site_id || a?.entity_ref; + if (eventSiteId && String(eventSiteId) !== String(siteId)) { + droppedForSiteMismatch += 1; + continue; + } + } + + const sev = String(a?.severity || '').toLowerCase(); + if (sev === 'critical' || sev === 'major' || sev === 'minor') { + counts[sev] += 1; + } + // `info` on Prisma events is often a NESTED OBJECT (e.g. `{ vpn_link_id: '...' }`), + // not a human-readable string — flatten defensively so the + // renderer doesn't dump `[object Object]` into chat. + const infoText = typeof a?.info === 'string' + ? a.info + : (a?.info ? JSON.stringify(a.info).slice(0, 200) : ''); + samples.push({ + code: a?.code || a?.category || a?.type || 'UNKNOWN', + message: infoText || a?.description || a?.message || '', + severity: sev || 'unknown', + ts: a?.time || a?.timestamp || a?._created_on_utc || null, + }); + } + samples.sort((a, b) => String(b.ts || '').localeCompare(String(a.ts || ''))); + return { + last1h: counts, + samples: samples.slice(0, 5), + // Diagnostic surface for the caller — lets sdwanEnrichment log + // "raw N events → K after site filter" so we can see whether + // Prisma's server-side site filter is working or being ignored. + _diag: { + rawEventCount: items.length, + droppedForSiteMismatch, + }, + }; +} + +function round(n, decimals) { + if (!Number.isFinite(n)) return null; + const p = Math.pow(10, decimals); + return Math.round(n * p) / p; +} diff --git a/services/renderers/phoneStatusRenderer.js b/services/renderers/phoneStatusRenderer.js index 2b20afe..9db4de3 100644 --- a/services/renderers/phoneStatusRenderer.js +++ b/services/renderers/phoneStatusRenderer.js @@ -35,10 +35,22 @@ import { simpleTimeAgo, formatBytes } from '../../utils/time.js'; * that a follow-up message with base-station diagnostics is on the * way. Chat handler passes this after the base count comes back * from discoverDectBases(); poller and HTTP callers pass 0. + * @param {boolean} [opts.wanFollowUpEnabled=false] + * When true, emits a "⏳ WAN metrics loading…" line just above the + * footer. Same rationale as `dectFollowUpBaseCount` — chat handler + * sets this only when the Prisma SD-WAN site resolves and a + * follow-up is genuinely in-flight; HTTP / Jira surfaces pass + * false and never see the line. * @returns {string} markdown, whitespace-trimmed and ready to send. */ export function renderPhoneStatusMarkdown(data, opts = {}) { - const { storeNum, detailed = false, footer = true, dectFollowUpBaseCount = 0 } = opts; + const { + storeNum, + detailed = false, + footer = true, + dectFollowUpBaseCount = 0, + wanFollowUpEnabled = false, + } = opts; let reply = `**Phone Status - Store ${storeNum}**\n\n`; @@ -211,6 +223,14 @@ export function renderPhoneStatusMarkdown(data, opts = {}) { reply += `_Detailed mode — additional fields above (use without ?detailed=true for compact view)_\n`; } + // WAN follow-up hint — same "silent for HTTP / Jira" rule as the + // DECT hint above. Sits at the bottom because WAN metrics are the + // last section chronologically (arrives after the DECT follow-up + // in most stores). + if (wanFollowUpEnabled) { + reply += `\n_⏳ WAN metrics loading from Prisma SD-WAN — a follow-up message will arrive shortly._\n`; + } + if (footer) { reply += `\n*Last checked: ${new Date().toLocaleTimeString()}*`; } diff --git a/services/renderers/voiceDiagRenderer.js b/services/renderers/voiceDiagRenderer.js index e018c5c..4b33c4b 100644 --- a/services/renderers/voiceDiagRenderer.js +++ b/services/renderers/voiceDiagRenderer.js @@ -50,6 +50,7 @@ const SEVERITY_LABEL = { * email?: string, * detailed?: boolean, * emitFooter?: boolean, + * wanWindowMinutes?: number, * }} [opts] * @returns {string} markdown, whitespace-trimmed */ @@ -60,6 +61,7 @@ export function renderVoiceDiagMarkdown(results, opts = {}) { email = null, detailed = false, emitFooter = true, + wanWindowMinutes = null, } = opts; const list = Array.isArray(results) ? results : []; @@ -77,7 +79,12 @@ export function renderVoiceDiagMarkdown(results, opts = {}) { } const counts = countBySeverity(list); - reply += summaryLine(counts) + '\n\n'; + reply += summaryLine(counts) + '\n'; + const hasWanCheck = list.some((r) => /^wan/i.test(r?.id || '')); + if (hasWanCheck && Number.isFinite(wanWindowMinutes)) { + reply += `_WAN window: ${humanWindow(wanWindowMinutes)}_\n`; + } + reply += '\n'; for (const severity of SEVERITY_ORDER) { const bucket = list.filter((r) => r.status === severity); @@ -91,7 +98,8 @@ export function renderVoiceDiagMarkdown(results, opts = {}) { for (const r of bucket) { reply += `- **${r.label}**: ${r.message}\n`; if (detailed && r.details && Object.keys(r.details).length > 0) { - reply += ` - ${renderDetails(r.details)}\n`; + const detailBlock = renderDetails(r.details); + if (detailBlock) reply += detailBlock + '\n'; } } reply += '\n'; @@ -131,18 +139,165 @@ function summaryLine({ error, warn, skipped, ok }) { return `Errors (${error}) · Warnings (${warn}) · Skipped (${skipped}) · OK (${ok})`; } -// Compact one-line rendering of the details object. Truncates arrays -// past a small length so the chat message doesn't blow up on -// verbose payloads (outgoingPermission's rule list, for example). -function renderDetails(details) { - if (details === null || details === undefined) return ''; - if (typeof details !== 'object') return String(details); +/** + * Human-friendly window label: 15m / 1h / 6h / 24h. + */ +function humanWindow(minutes) { + if (!Number.isFinite(minutes) || minutes <= 0) return `${minutes}m`; + if (minutes >= 1440 && minutes % 1440 === 0) return `${minutes / 1440}d`; + if (minutes >= 60 && minutes % 60 === 0) return `${minutes / 60}h`; + return `${minutes}m`; +} +/** + * Detail rendering. Recognises common check-result shapes and + * produces multiline markdown with icons/tables instead of a raw + * `key: value` dump — which for WAN checks in particular looks + * like an unreadable stringified JSON blob. + * + * Falls back to the compact key:value renderer for shapes we + * don't know how to format specially. Returns null when the + * details object contains nothing user-visible after formatting + * (e.g. WAN threshold constants that are already in the message). + * + * The shape detectors are ordered from most-specific to + * least-specific. Each returns a multiline string (with leading + * indent to nest under the parent bullet) or null to fall through. + */ +function renderDetails(details) { + if (details === null || details === undefined) return null; + if (typeof details !== 'object') return ` - ${String(details)}`; + + // ── Shape-aware formatters (best-fit wins) ──────────────────── + if (Array.isArray(details.perLink)) { + return renderPerLinkDetails(details); + } + if ('up' in details && 'down' in details && 'unknown' in details) { + return renderLinkStateDetails(details); + } + if ('critical' in details && 'major' in details && 'minor' in details) { + return renderAlarmsDetails(details); + } + if ('siteId' in details && 'siteName' in details) { + return renderSiteDetails(details); + } + if ('value' in details && 'warnThresh' in details && 'errorThresh' in details) { + return renderThresholdDetails(details); + } + + // ── Fallback: compact key:value dump ───────────────────────── const parts = []; for (const [k, v] of Object.entries(details)) { parts.push(`${k}: ${formatValue(v)}`); } - return parts.join(', '); + return parts.length > 0 ? ` - ${parts.join(', ')}` : null; +} + +const VERDICT_ICON = { ok: '✅', warn: '⚠️', error: '❌', unknown: '❓' }; + +function verdictIcon(v) { + return VERDICT_ICON[v] || '·'; +} + +/** + * WAN latency/jitter/loss/mos etc. Produces: + * + * - Threshold: warn > 150ms, error > 400ms + * - Per link: + * - ✅ Inet1-00782: 22.2 ms + * - ✅ Inet2-00782: 13.5 ms + * - ⚠️ 5G-LTE-00782: 165 ms + * - Roll-up: 3 total · 2 ok · 1 warn · 0 error + */ +function renderPerLinkDetails(details) { + const { perLink, warnThresh, errorThresh, standardLabel, total, ok, warn, error } = details; + const lines = []; + if (standardLabel) { + lines.push(` - Threshold: ${standardLabel}`); + } else if (Number.isFinite(warnThresh) && Number.isFinite(errorThresh)) { + lines.push(` - Threshold: warn @ ${warnThresh}, error @ ${errorThresh}`); + } + if (Array.isArray(perLink) && perLink.length > 0) { + lines.push(' - Per link:'); + for (const p of perLink) { + const val = p?.value == null || Number.isNaN(p.value) ? '—' : String(p.value); + lines.push(` - ${verdictIcon(p?.verdict)} ${p?.link}: ${val}`); + } + } + const rollup = [`${total ?? '?'} total`]; + if (Number.isFinite(ok)) rollup.push(`${ok} ok`); + if (Number.isFinite(warn)) rollup.push(`${warn} warn`); + if (Number.isFinite(error)) rollup.push(`${error} error`); + if (rollup.length > 1) lines.push(` - Roll-up: ${rollup.join(' · ')}`); + return lines.length > 0 ? lines.join('\n') : null; +} + +/** + * WAN Link State — {total, up, down, unknown, offenders, unknownLabels}. + */ +function renderLinkStateDetails(details) { + const { total, up, down, unknown, offenders = [], unknownLabels = [] } = details; + const lines = [` - Roll-up: ${total ?? '?'} total · ${up ?? 0} up · ${down ?? 0} down · ${unknown ?? 0} unknown`]; + if (offenders.length > 0) { + lines.push(` - Down: ${offenders.join(', ')}`); + } + if (unknownLabels.length > 0) { + lines.push(` - Unknown: ${unknownLabels.join(', ')}`); + } + return lines.join('\n'); +} + +/** + * Alarm counts — {critical, major, minor, recentSamples: [...]}. + */ +function renderAlarmsDetails(details) { + const { critical = 0, major = 0, minor = 0, recentSamples = [] } = details; + const lines = [` - Counts: 🔴 ${critical} critical · 🟠 ${major} major · 🟡 ${minor} minor`]; + if (Array.isArray(recentSamples) && recentSamples.length > 0) { + lines.push(' - Recent:'); + for (const a of recentSamples) { + const type = a?.type || a?.code || a?.alarm_type || 'unknown'; + const sev = a?.severity ? ` (${a.severity})` : ''; + lines.push(` - ${type}${sev}`); + } + } + return lines.join('\n'); +} + +/** + * WAN Site — {siteId, siteName, elementCount, connectedElementCount, linkCount}. + * Rendered as a compact one-liner since siteName is usually already + * in the message. + */ +function renderSiteDetails(details) { + const { siteId, elementCount, connectedElementCount, linkCount } = details; + const parts = []; + if (siteId) parts.push(`id: \`${siteId}\``); + if (Number.isFinite(elementCount)) { + parts.push( + `${elementCount} element(s)` + + (Number.isFinite(connectedElementCount) + ? ` (${connectedElementCount} connected)` + : ''), + ); + } + if (Number.isFinite(linkCount)) parts.push(`${linkCount} link(s)`); + return parts.length > 0 ? ` - ${parts.join(' · ')}` : null; +} + +/** + * Single value + thresholds — {value, warnThresh, errorThresh, breakdown}. + * The message already contains value + thresholds, so the details + * block just shows the sub-score breakdown (if any) and skips the + * redundant info. + */ +function renderThresholdDetails(details) { + const { breakdown } = details; + if (breakdown && typeof breakdown === 'object' && Object.keys(breakdown).length > 0) { + const parts = Object.entries(breakdown).map(([k, v]) => `${k}: ${v}`); + return ` - Breakdown: ${parts.join(' · ')}`; + } + return null; } function formatValue(v) { diff --git a/services/renderers/wanDiagnosticsRenderer.js b/services/renderers/wanDiagnosticsRenderer.js new file mode 100644 index 0000000..dc0d233 --- /dev/null +++ b/services/renderers/wanDiagnosticsRenderer.js @@ -0,0 +1,255 @@ +// src/services/renderers/wanDiagnosticsRenderer.js +// +// Pure markdown renderer for the /phonestatus WAN follow-up message. +// Takes a `collectSdwanForStore(storeNum)` result and produces the +// section that shows healthscore + per-path LQM + active alarms. +// +// Design mirrors renderDectDiagnosticsMarkdown in +// services/renderers/phoneStatusRenderer.js: +// - Pure function, no I/O. +// - Returns '' when there's genuinely nothing to say (caller +// no-ops on empty string). +// - Bullet lists only — Webex markdown doesn't render tables +// reliably, and the port-hygiene checks already prove bullets +// scan fine for per-device drilldowns. +// - Threshold-based icons (✅ good, ⚠️ warn, ❌ error, ❓ unknown) +// so a scanning operator can locate the bad link visually +// without reading numbers. + +// Thresholds are read from env at render time so the rendered +// icons stay in sync with the check bucket verdicts. Same defaults +// as services/voiceDiag/checks/wan/_helpers.js — kept in sync by +// convention (there's a regression test that pins them together). +function readThresholds() { + const num = (k, fallback) => { + const raw = Number(process.env[k]); + return Number.isFinite(raw) ? raw : fallback; + }; + return { + latencyWarn: num('WAN_STANDARD_LATENCY_WARN_MS', 150), + latencyError: num('WAN_STANDARD_LATENCY_ERROR_MS', 400), + jitterWarn: num('WAN_STANDARD_JITTER_WARN_MS', 30), + jitterError: num('WAN_STANDARD_JITTER_ERROR_MS', 50), + lossWarn: num('WAN_STANDARD_LOSS_WARN_PCT', 1), + lossError: num('WAN_STANDARD_LOSS_ERROR_PCT', 3), + mosWarn: num('WAN_STANDARD_MOS_WARN', 4.0), + mosError: num('WAN_STANDARD_MOS_ERROR', 3.5), + hsWarn: num('WAN_STANDARD_HEALTHSCORE_WARN', 80), + hsError: num('WAN_STANDARD_HEALTHSCORE_ERROR', 60), + }; +} + +/** + * Render a Prisma SD-WAN follow-up message. + * + * @param {object} data Result from collectSdwanForStore(storeNum). + * @param {object} [opts] + * @param {string} [opts.storeNum] used in the section header + * @param {boolean} [opts.footer=true] emit the "pulled at HH:MM:SS" footer + * @returns {string} markdown (or '' when there's nothing to render) + */ +export function renderWanDiagnosticsMarkdown(data, opts = {}) { + const { storeNum, footer = true } = opts; + if (!data || typeof data !== 'object') return ''; + + // No-site case: this is the "not a Prisma-managed store" happy + // path. We deliberately DON'T show a header for it — the follow-up + // caller only kicks the runner when discovery said yes, so seeing + // this branch here means something raced. Return ''. + if (!data.site) return ''; + + const t = readThresholds(); + + let out = `**WAN Diagnostics (Prisma SD-WAN) — Store ${storeNum || data.storeNum || '?'}**\n\n`; + + // Site + healthscore header line. + const hs = data.healthscore; + const hsIcon = hs?.value == null ? '❓' : iconFromNumeric(hs.value, t.hsError, t.hsWarn, /* lowIsBad */ true); + const hsText = hs?.value == null ? 'n/a' : `${hs.value}/100`; + const elCount = Array.isArray(data.elements) ? data.elements.length : 0; + const linkCount = Array.isArray(data.links) ? data.links.length : 0; + const windowLabel = data.window?.minutes ? ` • Window: ${humanWindowLabel(data.window.minutes)}` : ''; + out += + `Site: **${data.site.name}** (${elCount} element${elCount === 1 ? '' : 's'}, ` + + `${linkCount} WAN path${linkCount === 1 ? '' : 's'}) • Healthscore: ${hsIcon} ${hsText}` + + `${windowLabel}\n\n`; + + // Per-path bullet list. Ordered by "worst path first" so a + // scanning operator sees the offender at the top. + if (linkCount > 0) { + const ranked = [...data.links].sort((a, b) => rankLink(b, t) - rankLink(a, t)); + for (const link of ranked) { + out += renderOneLink(link, t); + } + } else { + out += `_No WAN path metrics available for this site._\n`; + } + + // Alarms summary (only if any are active). Samples are rolled up + // by (code + severity) so 20 identical NETWORK_ANYNETLINK_DOWN + // events show as one line with "×20" instead of pasting 20 nearly- + // identical JSON blobs into chat. Full alarm detail belongs in the + // Prisma UI — this is a summary surface. + const totalAlarms = + (data.alarms?.last1h?.critical || 0) + + (data.alarms?.last1h?.major || 0) + + (data.alarms?.last1h?.minor || 0); + if (totalAlarms > 0) { + const { critical = 0, major = 0, minor = 0 } = data.alarms.last1h; + const parts = []; + if (critical > 0) parts.push(`${critical} critical`); + if (major > 0) parts.push(`${major} major`); + if (minor > 0) parts.push(`${minor} minor`); + out += `\n🚨 Alarms (last 1h): ${parts.join(', ')}\n`; + + const rollups = rollupAlarms(data.alarms.samples || []); + for (const r of rollups.slice(0, 5)) { + const when = r.newestTs ? new Date(r.newestTs).toLocaleTimeString() : ''; + const count = r.count > 1 ? ` ×${r.count}` : ''; + out += + ` - ${sevIcon(r.severity)} \`${r.code}\`${count}` + + (when ? ` (most recent ${when})` : '') + + `\n`; + } + if (rollups.length > 5) { + out += ` - _+${rollups.length - 5} more alarm code${rollups.length - 5 === 1 ? '' : 's'}._\n`; + } + } + + // Per-metric fetch failures shown as small warnings so the operator + // knows the display is incomplete rather than "all clear". + if (Array.isArray(data.errors) && data.errors.length > 0) { + out += `\n_Partial fetch:_\n`; + for (const e of data.errors) { + out += ` - \`${e.scope}\` failed: ${e.message}\n`; + } + } + + if (footer) { + out += `\n*WAN metrics pulled at ${new Date().toLocaleTimeString()} from Prisma SD-WAN. Use \`/voicediag ${storeNum || data.storeNum} --only wanLatency,wanJitter,wanLoss,wanMos,wanHealthscore,wanLinkState,wanAlarms\` for pass/warn/error breakdowns.*`; + } + + return out.trim(); +} + +// ─── internals ───────────────────────────────────────────────────── + +/** + * Human-friendly window label: 15m / 1h / 6h / 24h. + * Mirrored from voiceDiagRenderer.js — kept in-file to avoid a + * tiny shared-utils import for a 5-line helper. + */ +function humanWindowLabel(minutes) { + if (!Number.isFinite(minutes) || minutes <= 0) return `${minutes}m`; + if (minutes >= 1440 && minutes % 1440 === 0) return `${minutes / 1440}d`; + if (minutes >= 60 && minutes % 60 === 0) return `${minutes / 60}h`; + return `${minutes}m`; +} + +function renderOneLink(link, t) { + const nameLabel = link.interfaceName || link.interfaceId; + const transport = link.transportType ? ` [${link.transportType}]` : ''; + const upIcon = link.up === null ? '❓' : (link.up ? '✅' : '❌'); + const upText = link.up === null ? 'unknown' : (link.up ? 'up' : 'DOWN'); + + let out = `- ${upIcon} **${nameLabel}**${transport} — ${upText}`; + + const parts = []; + parts.push(fmtMetric('latency', link.latencyMs, 'ms', t.latencyError, t.latencyWarn)); + parts.push(fmtMetric('jitter', link.jitterMs, 'ms', t.jitterError, t.jitterWarn)); + parts.push(fmtMetric('loss', link.lossPct, '%', t.lossError, t.lossWarn)); + parts.push(fmtMetric('MOS', link.mos, '', t.mosError, t.mosWarn, /* highIsGood */ true)); + + const filled = parts.filter(Boolean); + if (filled.length > 0) { + out += `\n ${filled.join(' • ')}`; + } + + out += '\n'; + return out; +} + +function fmtMetric(label, value, unit, errThresh, warnThresh, highIsGood = false) { + if (value === null || value === undefined) return ''; + const icon = iconFromNumeric(value, errThresh, warnThresh, /* lowIsBad */ highIsGood); + return `${icon} ${label} ${value}${unit}`; +} + +/** + * Icon selector. + * + * For most WAN metrics (latency/jitter/loss), HIGHER is worse. For + * MOS + healthscore, LOWER is worse. `lowIsBad` flips the sense. + * + * @param {number} value + * @param {number} errThresh numeric threshold for error severity + * @param {number} warnThresh numeric threshold for warn severity + * @param {boolean} lowIsBad true → low values trigger warn/error + */ +function iconFromNumeric(value, errThresh, warnThresh, lowIsBad = false) { + if (lowIsBad) { + if (value < errThresh) return '❌'; + if (value < warnThresh) return '⚠️'; + return '✅'; + } + if (value > errThresh) return '❌'; + if (value > warnThresh) return '⚠️'; + return '✅'; +} + +function sevIcon(sev) { + if (sev === 'critical') return '🔴'; + if (sev === 'major') return '🟠'; + if (sev === 'minor') return '🟡'; + return '⚪'; +} + +/** + * Roll alarm samples up by (code + severity). Preserves the newest + * timestamp per rollup and orders results critical → major → minor, + * then by count descending. This turns a wall of 20 near-identical + * NETWORK_ANYNETLINK_DOWN JSON blobs into one scannable line. + */ +function rollupAlarms(samples) { + const byKey = new Map(); + for (const s of samples) { + if (!s) continue; + const key = `${s.severity}::${s.code}`; + const prev = byKey.get(key); + if (prev) { + prev.count += 1; + if (!prev.newestTs || (s.ts && String(s.ts) > String(prev.newestTs))) { + prev.newestTs = s.ts; + } + } else { + byKey.set(key, { + code: s.code, + severity: s.severity, + count: 1, + newestTs: s.ts, + }); + } + } + const sevRank = { critical: 0, major: 1, minor: 2, unknown: 3 }; + return [...byKey.values()].sort((a, b) => { + const ra = sevRank[a.severity] ?? 9; + const rb = sevRank[b.severity] ?? 9; + if (ra !== rb) return ra - rb; + return b.count - a.count; + }); +} + +/** + * Rank a link on a 0-100 badness scale so the worst path floats to + * the top of the display. Adds contributions from each metric + * according to how far past the warn/error thresholds it is. + */ +function rankLink(link, t) { + let score = 0; + if (link.up === false) score += 100; + if (link.latencyMs != null && link.latencyMs > t.latencyWarn) score += link.latencyMs > t.latencyError ? 30 : 10; + if (link.jitterMs != null && link.jitterMs > t.jitterWarn) score += link.jitterMs > t.jitterError ? 20 : 5; + if (link.lossPct != null && link.lossPct > t.lossWarn) score += link.lossPct > t.lossError ? 25 : 8; + if (link.mos != null && link.mos < t.mosWarn) score += link.mos < t.mosError ? 25 : 8; + return score; +} diff --git a/services/voiceDiag/README.md b/services/voiceDiag/README.md index 14b49df..2b47a74 100644 --- a/services/voiceDiag/README.md +++ b/services/voiceDiag/README.md @@ -12,10 +12,18 @@ objects, and hands them to the renderer + adaptive-card layer in /voicediag default: hides OK checks, posts fixable cards /voicediag detail include OK checks + expand every details block /voicediag --only dnd,callForwarding +/voicediag --window 24h override the WAN look-back window (default 15m) /voicediag list-checks enumerate every registered check + its scope ``` -HTTP path: `GET /voicediag?storeNum=[&detailed=true][&only=dnd,callWaiting]`. +HTTP path: `GET /voicediag?storeNum=[&detailed=true][&only=dnd,callWaiting][&window=24h]`. + +`--window` accepts `Nm` / `Nh` / `Nd` shorthand (e.g. `15m`, `1h`, `6h`, +`24h`, `1d`) or a bare integer of minutes. Applies to the WAN +healthscore + LQM (per-path latency/jitter/loss/MOS) fetches. Alarms +are floored at 60m regardless — sub-hour alarm queries are usually +too noisy to be actionable. Global default is +`WAN_STANDARD_WINDOW_MINUTES` (env, defaults to 15). HTTP callers get the markdown snapshot only — remediation cards are chat-only. @@ -108,6 +116,87 @@ Both port-hygiene knobs live in `.env`: | `VOICE_STANDARD_PHONE_VLAN` | `102` | Expected VLAN for a store phone. Set per-site if the fleet moves onto a proper voice VLAN. | | `VOICE_STANDARD_ENABLED` | `true` | Global kill-switch for the port-hygiene bucket. `false` silences portType / portVlan / portPoe / portEnabled while Meraki cleanup is in progress. Feature-config checks always run. | +### WAN standards (Prisma SD-WAN) + +Sources Prisma SD-WAN metrics (see `integrations/paloalto/`) for +the store's site and grades per-path LQM + healthscore + alarms +against ITU-T G.114 / RFC 3550 references. Diagnostic-only — no +auto-remediation, since WAN config PUTs are the operator's job in +the Prisma portal, not the bot's. All checks skip cleanly with +"not applicable — missing sdwanSite" for non-Prisma-managed sites, +so the WAN bucket adds no noise to stores that live entirely +behind Meraki. + +Data flow: `voiceDiagService.buildContext()` calls +`collectSdwanForStore(storeNum)` in parallel with the Webex + +Meraki fetches. That composer resolves the store to a Prisma site +(`CG${pad5(storeNum)}` — e.g. store 782 → `CG00782`), pulls +elements + healthscore + LinkState + 4 LQM metrics + recent alarms +in parallel, and normalises to a stable shape the checks + the +`/phonestatus` follow-up renderer share. + +| Check | Standard | Non-compliant severity | Notes | +| ---------------- | --------------------------------------------------- | ----------------------------------------------- | ------------------------------------------------------------------------ | +| `wanSite` | Prisma site resolves for the store | skipped when unresolved | Info-only anchor — surfaces site + element + link counts | +| `wanHealthscore` | >= `WAN_STANDARD_HEALTHSCORE_WARN` (default 80) | warn < 80, **error** < 60 | Prisma AIOps composite (0-100) | +| `wanLinkState` | Every WAN path up | **error** if any path down | No remediation — physical / carrier work | +| `wanLatency` | <= `WAN_STANDARD_LATENCY_WARN_MS` (default 150ms) | warn > 150ms, **error** > 400ms | ITU-T G.114 one-way reference. Worst path drives severity. | +| `wanJitter` | <= `WAN_STANDARD_JITTER_WARN_MS` (default 30ms) | warn > 30ms, **error** > 50ms | RFC 3550 jitter-buffer envelope | +| `wanLoss` | <= `WAN_STANDARD_LOSS_WARN_PCT` (default 1%) | warn > 1%, **error** > 3% | G.711 PLC tolerance | +| `wanMos` | >= `WAN_STANDARD_MOS_WARN` (default 4.0) | warn < 4.0, **error** < 3.5 | ITU-T P.800 MOS scale. LOW is bad — the accessor flips the comparison. | +| `wanAlarms` | Zero critical + zero major alarms in the last hour | warn on major, **error** on critical | Minor alarms are info-only. Pass-through of Prisma severity. | + +Environment overrides (all optional — defaults match the standards +above): + +| Var | Default | Purpose | +| ------------------------------------- | ------- | ----------------------------------------------------------- | +| `WAN_STANDARD_LATENCY_WARN_MS` | `150` | Latency warn threshold, ms | +| `WAN_STANDARD_LATENCY_ERROR_MS` | `400` | Latency error threshold, ms | +| `WAN_STANDARD_JITTER_WARN_MS` | `30` | Jitter warn threshold, ms | +| `WAN_STANDARD_JITTER_ERROR_MS` | `50` | Jitter error threshold, ms | +| `WAN_STANDARD_LOSS_WARN_PCT` | `1` | Packet-loss warn threshold, % | +| `WAN_STANDARD_LOSS_ERROR_PCT` | `3` | Packet-loss error threshold, % | +| `WAN_STANDARD_MOS_WARN` | `4.0` | MOS warn threshold (LOW is bad) | +| `WAN_STANDARD_MOS_ERROR` | `3.5` | MOS error threshold | +| `WAN_STANDARD_HEALTHSCORE_WARN` | `80` | Site healthscore warn threshold (0-100) | +| `WAN_STANDARD_HEALTHSCORE_ERROR` | `60` | Site healthscore error threshold | +| `WAN_STANDARD_ENABLED` | `true` | Global kill-switch for the WAN bucket. `false` silences all 8 WAN checks while a Prisma cleanup / integration validation is in flight. | + +Prisma credentials and auth-mode selection live under the +`Palo Alto Prisma SD-WAN` block in `.env.example` (SASE OAuth 2.0 +recommended, legacy CloudGenix session-token supported for +backward compatibility). See that block for the full env +inventory. + +### Debugging Prisma schema drift + +Prisma SD-WAN's monitor API schemas vary between tenants — request +body shapes documented on pan.dev or in LIVEcommunity examples +frequently fail schema check on this tenant with cryptic 400 +messages like `"$.filter.wan_interfaces: is not defined in the +schema"` or `"$.view: does not have a value in the enumeration +[summary, timeseries]"`. Iterating on those in-band via +`/phonestatus` is painful (full DECT + Meraki + WAN pipeline runs +every attempt). + +Use the standalone probe script instead: + +``` +node scripts/prismaProbe.js discover 782 +node scripts/prismaProbe.js try-shapes health +node scripts/prismaProbe.js try-shapes lqm +node scripts/prismaProbe.js raw POST /sdwan/v3.7/api/events/query --body '{"limit":{"count":5}}' +``` + +`try-shapes` fires a curated list of candidate request bodies and +prints which pass schema check (with response-shape summary) and +which fail (with the exact SCHEMA_CHECK_FAIL field). When you find +a winning shape, port it into `integrations/paloalto/metrics.js` +and add a regression test in `tests/paloalto.metrics.test.js` so +the tenant's quirk doesn't get "helpfully cleaned up" by a future +refactor. Same auth stack as the bot, no side effects. + ## Apply-all-N-fixes card When two or more checks return fixable results, `/voicediag` diff --git a/services/voiceDiag/checks/index.js b/services/voiceDiag/checks/index.js index 3601d25..63472f6 100644 --- a/services/voiceDiag/checks/index.js +++ b/services/voiceDiag/checks/index.js @@ -43,11 +43,20 @@ import { portTypeCheck } from './port/portType.js'; import { portVlanCheck } from './port/portVlan.js'; import { portPoeCheck } from './port/portPoe.js'; import { portEnabledCheck } from './port/portEnabled.js'; +import { wanSiteCheck } from './wan/wanSite.js'; +import { wanHealthscoreCheck } from './wan/wanHealthscore.js'; +import { wanLinkStateCheck } from './wan/wanLinkState.js'; +import { wanLatencyCheck } from './wan/wanLatency.js'; +import { wanJitterCheck } from './wan/wanJitter.js'; +import { wanLossCheck } from './wan/wanLoss.js'; +import { wanMosCheck } from './wan/wanMos.js'; +import { wanAlarmsCheck } from './wan/wanAlarms.js'; // Order: user-facing feature signals first (things an operator can -// see from the phone UI), then network-side port hygiene, then the -// broad-brush online summary last so it acts as a reachability -// closer. +// see from the phone UI), then LAN-side port hygiene, then the WAN +// bucket (Prisma SD-WAN — the "network edge" one hop out from the +// LAN), then the broad-brush online summary last so it acts as a +// reachability closer. export const CHECKS = [ dndCheck, callForwardingCheck, @@ -61,6 +70,14 @@ export const CHECKS = [ portVlanCheck, portPoeCheck, portEnabledCheck, + wanSiteCheck, + wanHealthscoreCheck, + wanLinkStateCheck, + wanLatencyCheck, + wanJitterCheck, + wanLossCheck, + wanMosCheck, + wanAlarmsCheck, phoneOnlineCheck, ]; diff --git a/services/voiceDiag/checks/wan/_helpers.js b/services/voiceDiag/checks/wan/_helpers.js new file mode 100644 index 0000000..01405b8 --- /dev/null +++ b/services/voiceDiag/checks/wan/_helpers.js @@ -0,0 +1,213 @@ +// src/services/voiceDiag/checks/wan/_helpers.js +// +// Shared kill-switch + threshold accessors for the /voicediag WAN +// bucket. Mirrors services/voiceDiag/checks/port/_helpers.js in +// intent — one file per check for the actual logic, this file for +// the common env-reading and formatting boilerplate. +// +// Threshold accessors read `process.env` at *call time* rather than +// at import time. That's what lets tests do `process.env.X = '...'` +// + await the check without needing a module cache reset. The +// downside is a couple of extra env reads per run, which is nothing +// against the network work each WAN check does (or doesn't do — the +// checks reuse data already fetched during buildContext()). + +/** + * Global kill-switch for the WAN bucket. Env + * `WAN_STANDARD_ENABLED=false` returns a short "WAN checks are + * currently disabled" skip result that every WAN check can return + * verbatim. Feature-config + port-hygiene checks are intentionally + * *not* gated by this — this only silences the SD-WAN bucket while + * the underlying Prisma tenant is being reconfigured or the + * integration is being validated. + * + * Same contract as port/_helpers.js:maybeSkippedByKillSwitch — + * returns null when enabled (caller proceeds) or a full CheckResult + * with `status: 'skipped'` when disabled (caller returns it as-is). + * + * @param {object} check the check descriptor (used for label only) + * @returns {null | {status:'skipped', message:string, details:object, remediation:null}} + */ +export function maybeSkippedByKillSwitch(check) { + const raw = String(process.env.WAN_STANDARD_ENABLED ?? 'true').toLowerCase().trim(); + const enabled = !(raw === 'false' || raw === '0' || raw === 'no' || raw === 'off'); + if (enabled) return null; + return { + status: 'skipped', + message: + `${check.label} skipped — WAN_STANDARD_ENABLED=false (SD-WAN checks are silenced).`, + details: { killSwitch: 'WAN_STANDARD_ENABLED', value: raw }, + remediation: null, + }; +} + +// ─── Threshold accessors ────────────────────────────────────────── +// +// Defaults match the ITU-T G.114 / RFC 3550 references for voice +// quality. Any of these can be overridden per-tenant via env; the +// renderer + the checks read from the same accessors so the icon +// in the phonestatus follow-up always agrees with the verdict in +// voicediag. + +function num(envKey, fallback) { + const raw = Number(process.env[envKey]); + return Number.isFinite(raw) ? raw : fallback; +} + +// Latency (one-way estimate, milliseconds) +export const getLatencyWarnMs = () => num('WAN_STANDARD_LATENCY_WARN_MS', 150); +export const getLatencyErrorMs = () => num('WAN_STANDARD_LATENCY_ERROR_MS', 400); + +// Jitter (inter-packet arrival variation, milliseconds) +export const getJitterWarnMs = () => num('WAN_STANDARD_JITTER_WARN_MS', 30); +export const getJitterErrorMs = () => num('WAN_STANDARD_JITTER_ERROR_MS', 50); + +// Packet loss (percent 0-100) +export const getLossWarnPct = () => num('WAN_STANDARD_LOSS_WARN_PCT', 1); +export const getLossErrorPct = () => num('WAN_STANDARD_LOSS_ERROR_PCT', 3); + +// MOS (Mean Opinion Score, 1.0-5.0 — HIGHER is better, so warn/err +// mean "value FALLS BELOW this") +export const getMosWarn = () => num('WAN_STANDARD_MOS_WARN', 4.0); +export const getMosError = () => num('WAN_STANDARD_MOS_ERROR', 3.5); + +// Healthscore (Prisma AIOps roll-up, 0-100 — HIGHER is better) +export const getHealthscoreWarn = () => num('WAN_STANDARD_HEALTHSCORE_WARN', 80); +export const getHealthscoreError = () => num('WAN_STANDARD_HEALTHSCORE_ERROR', 60); + +// ─── Utilities used by multiple checks ──────────────────────────── + +/** + * Safe number-of-links accessor. WAN checks that grade the whole + * site (healthscore, alarms) don't care about link count, but the + * per-link checks below use it to short-circuit when there are no + * links at all (returns skipped with an actionable message). + */ +export function getLinks(ctx) { + const raw = ctx?.sdwanData?.links; + return Array.isArray(raw) ? raw : []; +} + +/** Label a link for messages. Kept short so aggregate messages + * don't blow past Webex's chat readability. */ +export function labelForLink(link) { + const name = link?.interfaceName || link?.interfaceId || 'link'; + const el = link?.elementName ? `@${link.elementName}` : ''; + const tx = link?.transportType ? ` [${link.transportType}]` : ''; + return `${name}${el}${tx}`; +} + +/** + * Evaluate a numeric per-link metric against warn/error thresholds + * across every path, returning a CheckResult. Worst path drives + * severity — a single bad link takes precedence over three good + * ones because "the phone routed over the bad path sounds terrible" + * is the ticket we're trying to catch. + * + * Consolidated here so the four LQM checks (latency / jitter / + * loss / mos) don't duplicate 40 lines of comparison logic — they + * pass config in, get a CheckResult out. + * + * @param {object} args + * @param {Array} args.links ctx.sdwanData.links + * @param {string} args.metricKey row property to read (e.g. 'latencyMs') + * @param {string} args.label label for messages (e.g. 'latency') + * @param {string} args.unit display unit (e.g. 'ms', '%', '') + * @param {number} args.warnThresh + * @param {number} args.errorThresh + * @param {boolean} args.lowIsBad true → warn/error when value FALLS BELOW threshold (MOS) + * @param {string} args.standardLabel for the "compliant range" phrasing + */ +export function evaluatePerLinkMetric({ + links, + metricKey, + label, + unit, + warnThresh, + errorThresh, + lowIsBad = false, + standardLabel, +}) { + if (!Array.isArray(links) || links.length === 0) { + return { + status: 'skipped', + message: `No WAN paths reported for this site.`, + details: null, + remediation: null, + }; + } + + const withValue = links.filter((l) => Number.isFinite(l[metricKey])); + if (withValue.length === 0) { + return { + status: 'skipped', + message: `${label} not available from Prisma for any path (partial fetch).`, + details: null, + remediation: null, + }; + } + + const grade = (v) => { + if (lowIsBad) { + if (v < errorThresh) return 'error'; + if (v < warnThresh) return 'warn'; + return 'ok'; + } + if (v > errorThresh) return 'error'; + if (v > warnThresh) return 'warn'; + return 'ok'; + }; + + const perLink = withValue.map((l) => ({ + link: labelForLink(l), + value: l[metricKey], + interfaceId: l.interfaceId, + verdict: grade(l[metricKey]), + })); + + const errors = perLink.filter((r) => r.verdict === 'error'); + const warns = perLink.filter((r) => r.verdict === 'warn'); + + const worstLine = (rows) => + rows.map((r) => `${r.link} (${r.value}${unit})`).join(', '); + + const details = { + total: perLink.length, + ok: perLink.filter((r) => r.verdict === 'ok').length, + warn: warns.length, + error: errors.length, + perLink, + warnThresh, + errorThresh, + standardLabel, + }; + + if (errors.length > 0) { + return { + status: 'error', + message: + `${errors.length} path(s) with ${label} past error threshold ` + + `(${standardLabel}): ${worstLine(errors)}.`, + details, + remediation: null, + }; + } + + if (warns.length > 0) { + return { + status: 'warn', + message: + `${warns.length} path(s) with elevated ${label} ` + + `(${standardLabel}): ${worstLine(warns)}.`, + details, + remediation: null, + }; + } + + return { + status: 'ok', + message: `All ${perLink.length} path(s) with ${label} in the compliant range (${standardLabel}).`, + details, + remediation: null, + }; +} diff --git a/services/voiceDiag/checks/wan/wanAlarms.js b/services/voiceDiag/checks/wan/wanAlarms.js new file mode 100644 index 0000000..2bf8240 --- /dev/null +++ b/services/voiceDiag/checks/wan/wanAlarms.js @@ -0,0 +1,90 @@ +// src/services/voiceDiag/checks/wan/wanAlarms.js +// +// Recent Prisma alarms surface. Not a threshold check — the +// severity is a direct pass-through of what Prisma raised: +// +// - critical → error +// - major → warn +// - minor → info-only (stays 'ok', but details show the count) +// +// Window defaults to the composer's 1-hour lookback. An "active +// incident" that Prisma sees is materially more urgent than a +// snapshot LQM anomaly, so this deliberately runs even when the +// per-metric checks are all green — a critical alarm from 45 min +// ago is still the operator's problem. + +import { maybeSkippedByKillSwitch } from './_helpers.js'; + +export const WAN_ALARMS_STANDARDS = Object.freeze({ + criticalAllowed: 0, + majorAllowed: 0, +}); + +export const wanAlarmsCheck = { + id: 'wanAlarms', + label: 'SD-WAN Alarms (last 1h)', + requires: ['sdwanSite'], + scope: null, + standards: WAN_ALARMS_STANDARDS, + + async run(ctx) { + const skip = maybeSkippedByKillSwitch(wanAlarmsCheck); + if (skip) return skip; + + const alarms = ctx.sdwanData?.alarms; + if (!alarms) { + return { + status: 'skipped', + message: 'Alarms feed not available from Prisma (partial fetch).', + details: null, + remediation: null, + }; + } + + const { critical = 0, major = 0, minor = 0 } = alarms.last1h || {}; + const samples = Array.isArray(alarms.samples) ? alarms.samples : []; + + const details = { + critical, major, minor, + recentSamples: samples.slice(0, 3), + }; + + if (critical > 0) { + return { + status: 'error', + message: + `${critical} critical alarm${critical === 1 ? '' : 's'} raised in the last hour` + + (samples[0]?.message ? ` — most recent: ${samples[0].code}: ${samples[0].message}` : '.'), + details, + remediation: null, + }; + } + + if (major > 0) { + return { + status: 'warn', + message: + `${major} major alarm${major === 1 ? '' : 's'} raised in the last hour` + + (samples[0]?.message ? ` — most recent: ${samples[0].code}: ${samples[0].message}` : '.'), + details, + remediation: null, + }; + } + + if (minor > 0) { + return { + status: 'ok', + message: `${minor} minor alarm${minor === 1 ? '' : 's'} in the last hour, no critical or major.`, + details, + remediation: null, + }; + } + + return { + status: 'ok', + message: 'No alarms raised in the last hour.', + details, + remediation: null, + }; + }, +}; diff --git a/services/voiceDiag/checks/wan/wanHealthscore.js b/services/voiceDiag/checks/wan/wanHealthscore.js new file mode 100644 index 0000000..31adceb --- /dev/null +++ b/services/voiceDiag/checks/wan/wanHealthscore.js @@ -0,0 +1,77 @@ +// src/services/voiceDiag/checks/wan/wanHealthscore.js +// +// Prisma AIOps site healthscore (0-100, higher is better). This is +// Prisma's own composite score that rolls up per-path LQM, link +// availability, application health, and device health — so it's a +// good broad-brush signal for "is anything wrong on this site" +// before drilling into the individual per-path checks below. +// +// Thresholds default to 80 (warn) / 60 (error), reflecting the +// bands Prisma's own UI uses in its site-health dashboard. Both +// are overridable via WAN_STANDARD_HEALTHSCORE_WARN / _ERROR. +// +// Skipped when no site was matched OR when the healthscore fetch +// failed (composer preserves that in ctx.sdwanData.errors[]). + +import { + maybeSkippedByKillSwitch, + getHealthscoreWarn, + getHealthscoreError, +} from './_helpers.js'; + +export const WAN_HEALTHSCORE_STANDARDS = Object.freeze({ + minWarn: 80, + minError: 60, +}); + +export const wanHealthscoreCheck = { + id: 'wanHealthscore', + label: 'SD-WAN Site Healthscore', + requires: ['sdwanSite'], + scope: null, + standards: WAN_HEALTHSCORE_STANDARDS, + + async run(ctx) { + const skip = maybeSkippedByKillSwitch(wanHealthscoreCheck); + if (skip) return skip; + + const hs = ctx.sdwanData?.healthscore; + if (!hs || hs.value === null || hs.value === undefined) { + return { + status: 'skipped', + message: 'Healthscore not available from Prisma (likely a partial fetch — see other checks).', + details: null, + remediation: null, + }; + } + + const value = Number(hs.value); + const warnThresh = getHealthscoreWarn(); + const errorThresh = getHealthscoreError(); + + if (value < errorThresh) { + return { + status: 'error', + message: `Healthscore ${value}/100 is below the error floor (${errorThresh}). The site is in a degraded state per Prisma AIOps.`, + details: { value, warnThresh, errorThresh, breakdown: hs.breakdown || {} }, + remediation: null, + }; + } + + if (value < warnThresh) { + return { + status: 'warn', + message: `Healthscore ${value}/100 is below the warning threshold (${warnThresh}). Investigate before it becomes user-visible.`, + details: { value, warnThresh, errorThresh, breakdown: hs.breakdown || {} }, + remediation: null, + }; + } + + return { + status: 'ok', + message: `Healthscore ${value}/100 (>=${warnThresh}). Site is healthy per Prisma AIOps.`, + details: { value, warnThresh, errorThresh, breakdown: hs.breakdown || {} }, + remediation: null, + }; + }, +}; diff --git a/services/voiceDiag/checks/wan/wanJitter.js b/services/voiceDiag/checks/wan/wanJitter.js new file mode 100644 index 0000000..f74c141 --- /dev/null +++ b/services/voiceDiag/checks/wan/wanJitter.js @@ -0,0 +1,51 @@ +// src/services/voiceDiag/checks/wan/wanJitter.js +// +// Per-path WAN jitter (LQM inter-packet arrival variation, ms) vs +// voice-quality references: +// - <= 30ms : good (jitter buffer handles it cleanly) +// - 30-50ms : warn (audible choppiness on longer calls) +// - > 50ms : error (jitter buffer starts dropping packets; +// garbled audio) +// +// Both thresholds are env-configurable via WAN_STANDARD_JITTER_*. + +import { + maybeSkippedByKillSwitch, + getLinks, + getJitterWarnMs, + getJitterErrorMs, + evaluatePerLinkMetric, +} from './_helpers.js'; + +export const WAN_JITTER_STANDARDS = Object.freeze({ + maxWarnMs: 30, + maxErrorMs: 50, + unit: 'ms', + reference: 'RFC 3550 jitter buffer sizing', +}); + +export const wanJitterCheck = { + id: 'wanJitter', + label: 'SD-WAN Jitter (per path)', + requires: ['sdwanSite'], + scope: null, + standards: WAN_JITTER_STANDARDS, + + async run(ctx) { + const skip = maybeSkippedByKillSwitch(wanJitterCheck); + if (skip) return skip; + + const warnThresh = getJitterWarnMs(); + const errorThresh = getJitterErrorMs(); + return evaluatePerLinkMetric({ + links: getLinks(ctx), + metricKey: 'jitterMs', + label: 'jitter', + unit: 'ms', + warnThresh, + errorThresh, + lowIsBad: false, + standardLabel: `warn > ${warnThresh}ms, error > ${errorThresh}ms`, + }); + }, +}; diff --git a/services/voiceDiag/checks/wan/wanLatency.js b/services/voiceDiag/checks/wan/wanLatency.js new file mode 100644 index 0000000..4852599 --- /dev/null +++ b/services/voiceDiag/checks/wan/wanLatency.js @@ -0,0 +1,58 @@ +// src/services/voiceDiag/checks/wan/wanLatency.js +// +// Per-path WAN latency (LQM one-way average, milliseconds) vs +// ITU-T G.114 references: +// - <= 150ms : good (no perceptible degradation) +// - 150-400ms : warn (audible echo/lag, still usable) +// - > 400ms : error (voice becomes duplex-half, users stop +// talking over each other and start waiting) +// +// Both thresholds are env-configurable via WAN_STANDARD_LATENCY_* +// (see .env.example). Worst path drives severity — a single bad +// path takes precedence over three good ones because that's the +// one your voice ticket is routed over. + +import { + maybeSkippedByKillSwitch, + getLinks, + getLatencyWarnMs, + getLatencyErrorMs, + evaluatePerLinkMetric, +} from './_helpers.js'; + +// Standards frozen as a plain constant so the README standards +// table + regression guards can read it without invoking the +// accessors. Same values as `getLatencyWarnMs()` / `getLatencyErrorMs()` +// unless the operator overrode via env. +export const WAN_LATENCY_STANDARDS = Object.freeze({ + maxWarnMs: 150, + maxErrorMs: 400, + unit: 'ms', + reference: 'ITU-T G.114', +}); + +export const wanLatencyCheck = { + id: 'wanLatency', + label: 'SD-WAN Latency (per path)', + requires: ['sdwanSite'], + scope: null, + standards: WAN_LATENCY_STANDARDS, + + async run(ctx) { + const skip = maybeSkippedByKillSwitch(wanLatencyCheck); + if (skip) return skip; + + const warnThresh = getLatencyWarnMs(); + const errorThresh = getLatencyErrorMs(); + return evaluatePerLinkMetric({ + links: getLinks(ctx), + metricKey: 'latencyMs', + label: 'latency', + unit: 'ms', + warnThresh, + errorThresh, + lowIsBad: false, + standardLabel: `warn > ${warnThresh}ms, error > ${errorThresh}ms`, + }); + }, +}; diff --git a/services/voiceDiag/checks/wan/wanLinkState.js b/services/voiceDiag/checks/wan/wanLinkState.js new file mode 100644 index 0000000..76c5999 --- /dev/null +++ b/services/voiceDiag/checks/wan/wanLinkState.js @@ -0,0 +1,92 @@ +// src/services/voiceDiag/checks/wan/wanLinkState.js +// +// Per-path administrative + operational up/down state. Any single +// path being down at a Prisma SD-WAN site is an error-severity +// signal: SD-WAN by design tolerates one path being down (traffic +// shifts to a surviving path), but a DOWN path is still lost +// capacity + a lost failover buffer, and voice-quality problems +// are much more likely on the remaining paths under load. +// +// A path with `up === null` (LinkState metric didn't come back) +// isn't counted as down — surfaced as "unknown" in details so the +// operator can see the visibility gap. +// +// No remediation — bringing a WAN link back up is a physical / +// carrier task, not something the bot should try to automate. + +import { + maybeSkippedByKillSwitch, + getLinks, + labelForLink, +} from './_helpers.js'; + +export const WAN_LINK_STATE_STANDARDS = Object.freeze({ + perLinkUp: true, +}); + +export const wanLinkStateCheck = { + id: 'wanLinkState', + label: 'SD-WAN Link State', + requires: ['sdwanSite'], + scope: null, + standards: WAN_LINK_STATE_STANDARDS, + + async run(ctx) { + const skip = maybeSkippedByKillSwitch(wanLinkStateCheck); + if (skip) return skip; + + const links = getLinks(ctx); + if (links.length === 0) { + return { + status: 'skipped', + message: 'No WAN paths reported for this site.', + details: null, + remediation: null, + }; + } + + const down = links.filter((l) => l.up === false); + const unknown = links.filter((l) => l.up === null || l.up === undefined); + const up = links.filter((l) => l.up === true); + + const details = { + total: links.length, + up: up.length, + down: down.length, + unknown: unknown.length, + offenders: down.map(labelForLink), + unknownLabels: unknown.map(labelForLink), + }; + + if (down.length > 0) { + return { + status: 'error', + message: + `${down.length} of ${links.length} WAN path(s) DOWN: ` + + down.map(labelForLink).join(', ') + + `. Voice quality on surviving paths may degrade under load.`, + details, + remediation: null, + }; + } + + if (unknown.length > 0 && up.length === 0) { + // Prisma returned NO LinkState samples at all — treat as a + // reporting warning, not a health error. + return { + status: 'warn', + message: `Link state unknown for all ${unknown.length} WAN path(s) (no samples from Prisma).`, + details, + remediation: null, + }; + } + + return { + status: 'ok', + message: `All ${up.length} WAN path(s) up.` + + (unknown.length > 0 ? ` (${unknown.length} with no state samples.)` : ''), + details, + remediation: null, + }; + }, +}; diff --git a/services/voiceDiag/checks/wan/wanLoss.js b/services/voiceDiag/checks/wan/wanLoss.js new file mode 100644 index 0000000..72ed573 --- /dev/null +++ b/services/voiceDiag/checks/wan/wanLoss.js @@ -0,0 +1,50 @@ +// src/services/voiceDiag/checks/wan/wanLoss.js +// +// Per-path WAN packet loss (LQM %, 0-100). Voice codec loss +// tolerance: +// - <= 1% : good (PLC hides it) +// - 1-3% : warn (audible artifacts on longer calls) +// - > 3% : error (words dropping, users repeating themselves) +// +// Both thresholds are env-configurable via WAN_STANDARD_LOSS_*. + +import { + maybeSkippedByKillSwitch, + getLinks, + getLossWarnPct, + getLossErrorPct, + evaluatePerLinkMetric, +} from './_helpers.js'; + +export const WAN_LOSS_STANDARDS = Object.freeze({ + maxWarnPct: 1, + maxErrorPct: 3, + unit: '%', + reference: 'G.711 PLC tolerance envelope', +}); + +export const wanLossCheck = { + id: 'wanLoss', + label: 'SD-WAN Packet Loss (per path)', + requires: ['sdwanSite'], + scope: null, + standards: WAN_LOSS_STANDARDS, + + async run(ctx) { + const skip = maybeSkippedByKillSwitch(wanLossCheck); + if (skip) return skip; + + const warnThresh = getLossWarnPct(); + const errorThresh = getLossErrorPct(); + return evaluatePerLinkMetric({ + links: getLinks(ctx), + metricKey: 'lossPct', + label: 'packet loss', + unit: '%', + warnThresh, + errorThresh, + lowIsBad: false, + standardLabel: `warn > ${warnThresh}%, error > ${errorThresh}%`, + }); + }, +}; diff --git a/services/voiceDiag/checks/wan/wanMos.js b/services/voiceDiag/checks/wan/wanMos.js new file mode 100644 index 0000000..8de284f --- /dev/null +++ b/services/voiceDiag/checks/wan/wanMos.js @@ -0,0 +1,55 @@ +// src/services/voiceDiag/checks/wan/wanMos.js +// +// Per-path Mean Opinion Score (MOS, 1.0-5.0). MOS is Prisma's +// composite quality score derived from LQM latency+jitter+loss; +// it's the number to correlate with user-reported "voice sounds +// bad" tickets because it factors all the physical-layer signals +// into one number. +// +// - >= 4.0 : good ("toll quality") +// - 3.5-4.0: warn ("acceptable, users notice") +// - < 3.5 : error ("degraded, users struggle") +// +// Unlike latency/jitter/loss, MOS is inverted: LOW is bad. The +// shared evaluator honours the `lowIsBad: true` flag. + +import { + maybeSkippedByKillSwitch, + getLinks, + getMosWarn, + getMosError, + evaluatePerLinkMetric, +} from './_helpers.js'; + +export const WAN_MOS_STANDARDS = Object.freeze({ + minWarn: 4.0, + minError: 3.5, + unit: '', + reference: 'ITU-T P.800 MOS scale', +}); + +export const wanMosCheck = { + id: 'wanMos', + label: 'SD-WAN MOS (per path)', + requires: ['sdwanSite'], + scope: null, + standards: WAN_MOS_STANDARDS, + + async run(ctx) { + const skip = maybeSkippedByKillSwitch(wanMosCheck); + if (skip) return skip; + + const warnThresh = getMosWarn(); + const errorThresh = getMosError(); + return evaluatePerLinkMetric({ + links: getLinks(ctx), + metricKey: 'mos', + label: 'MOS', + unit: '', + warnThresh, + errorThresh, + lowIsBad: true, + standardLabel: `warn < ${warnThresh}, error < ${errorThresh}`, + }); + }, +}; diff --git a/services/voiceDiag/checks/wan/wanSite.js b/services/voiceDiag/checks/wan/wanSite.js new file mode 100644 index 0000000..cc8df6f --- /dev/null +++ b/services/voiceDiag/checks/wan/wanSite.js @@ -0,0 +1,57 @@ +// src/services/voiceDiag/checks/wan/wanSite.js +// +// Info-only anchor check for the WAN bucket. Confirms which Prisma +// SD-WAN site the store resolved to and how many appliances / +// links we're seeing. Doesn't grade anything — its whole purpose is +// to give the operator context ("yes we found the site; here's what +// we're seeing") before the per-signal checks below start emitting +// warn/error verdicts. +// +// Skipped when no Prisma site was matched — the runner shows that +// as `skipped: not applicable — missing sdwanSite`, which is more +// honest than a hard-coded "we couldn't reach Prisma" error +// because there are two genuine reasons the site is null: (1) +// this store isn't Prisma-managed at all, or (2) the Prisma +// integration is misconfigured. buildContext() logs which one it +// was. + +import { maybeSkippedByKillSwitch } from './_helpers.js'; + +export const WAN_SITE_STANDARDS = Object.freeze({ + // No thresholds — this is a discovery/context check. + requiresPrismaSite: true, +}); + +export const wanSiteCheck = { + id: 'wanSite', + label: 'SD-WAN Site Discovery', + requires: ['sdwanSite'], + scope: null, + standards: WAN_SITE_STANDARDS, + + async run(ctx) { + const skip = maybeSkippedByKillSwitch(wanSiteCheck); + if (skip) return skip; + + const site = ctx.sdwanSite; + const elements = Array.isArray(ctx.sdwanData?.elements) ? ctx.sdwanData.elements : []; + const links = Array.isArray(ctx.sdwanData?.links) ? ctx.sdwanData.links : []; + + const connectedCount = elements.filter((e) => e.connected).length; + + return { + status: 'ok', + message: + `Site **${site.name}** (${elements.length} element${elements.length === 1 ? '' : 's'}, ` + + `${connectedCount} connected) with ${links.length} WAN path${links.length === 1 ? '' : 's'}.`, + details: { + siteId: site.id, + siteName: site.name, + elementCount: elements.length, + connectedElementCount: connectedCount, + linkCount: links.length, + }, + remediation: null, + }; + }, +}; diff --git a/services/voiceDiag/voiceDiagService.js b/services/voiceDiag/voiceDiagService.js index 1d8c25f..7546a8f 100644 --- a/services/voiceDiag/voiceDiagService.js +++ b/services/voiceDiag/voiceDiagService.js @@ -56,9 +56,15 @@ import { CHECKS, getCheckById } from './checks/index.js'; * personLabel: string, * }>} */ -export async function buildContext(storeNum) { +export async function buildContext(storeNum, opts = {}) { const email = `ae${String(storeNum).padStart(5, '0')}@ae.com`; - logger('voicediag', `Building context for store ${storeNum} → ${email}`, 'debug'); + const windowMinutes = opts.windowMinutes; + logger( + 'voicediag', + `Building context for store ${storeNum} → ${email}` + + (windowMinutes ? ` (WAN window: ${windowMinutes}m)` : ''), + 'debug', + ); const { default: webex } = await import('../../integrations/webex/WebexClient.js'); const { @@ -67,33 +73,52 @@ export async function buildContext(storeNum) { getTelephonyProfile, collectPhoneStatus, } = await import('../phoneService.js'); + // Prisma SD-WAN enrichment is lazy for the same reason WebexClient + // is: it reads process.env at import time (auth mode selection) + // and unit tests should be able to run without provisioning any + // Prisma creds. Fetch failures are already absorbed inside the + // composer, so the wrapping try/catch here is a belt-and-braces + // guard against unexpected import-time surprises only. + let collectSdwanForStore; + try { + ({ collectSdwanForStore } = await import('../enrichment/sdwanEnrichment.js')); + } catch (err) { + logger('voicediag', `Prisma enrichment unavailable: ${err.message}`, 'warn'); + collectSdwanForStore = null; + } const personId = await getPersonIdByEmail(email); // If we have no person, skip the expensive phoneStatus fetch — the // renderer will show every check as `skipped: no user`. This keeps - // the "unknown store" case cheap. - const [personRes, telProfRes, phoneStatusRes] = personId - ? await Promise.allSettled([ - getPersonDetails(personId), - getTelephonyProfile(personId), - collectPhoneStatus(storeNum).catch((err) => { + // the "unknown store" case cheap. We still fetch SD-WAN data — a + // store with no Webex user can absolutely still have a WAN we care + // about (dark store investigation, e.g. the /findEmptyLocations + // follow-up). + const [personRes, telProfRes, phoneStatusRes, sdwanRes] = await Promise.allSettled([ + personId ? getPersonDetails(personId) : Promise.resolve(null), + personId ? getTelephonyProfile(personId) : Promise.resolve({}), + personId + ? collectPhoneStatus(storeNum).catch((err) => { // collectPhoneStatus can throw noisily on network hiccups. // We tolerate this: the phoneOnline check will read // ctx.phoneStatus and show 'skipped' if it's null. logger('voicediag', `collectPhoneStatus soft-failed: ${err.message}`, 'warn'); return null; - }), - ]) - : [ - { status: 'fulfilled', value: null }, - { status: 'fulfilled', value: {} }, - { status: 'fulfilled', value: null }, - ]; + }) + : Promise.resolve(null), + collectSdwanForStore + ? collectSdwanForStore(storeNum, { windowMinutes }).catch((err) => { + logger('voicediag', `collectSdwanForStore soft-failed: ${err.message}`, 'warn'); + return null; + }) + : Promise.resolve(null), + ]); const person = personRes.status === 'fulfilled' ? personRes.value : null; const telephonyProfile = telProfRes.status === 'fulfilled' ? (telProfRes.value || {}) : {}; const phoneStatus = phoneStatusRes.status === 'fulfilled' ? phoneStatusRes.value : null; + const sdwanData = sdwanRes.status === 'fulfilled' ? sdwanRes.value : null; const personLabel = person?.displayName || email; @@ -112,6 +137,12 @@ export async function buildContext(storeNum) { // stub context with a fake webex; no ctx passes through the real // client unless it came from buildContext(). webex, + // Prisma SD-WAN data. sdwanData is the full composer output; + // sdwanSite is the shortcut checks use to gate `requires: + // 'sdwanSite'`. Both null when this store isn't Prisma-managed + // (or when the Prisma integration hasn't been configured). + sdwanData, + sdwanSite: sdwanData?.site || null, }; } @@ -182,6 +213,12 @@ function missingRequirements(check, ctx) { else if (req === 'telephonyProfile' && (!ctx.telephonyProfile || Object.keys(ctx.telephonyProfile).length === 0)) { missing.push('telephonyProfile'); } + // WAN bucket vocabulary. `sdwanSite` gates on "is this a Prisma- + // managed store at all"; `sdwanData` gates on "did the composer + // produce anything usable" (i.e. discriminate a network-failure + // day from a non-Prisma store). + else if (req === 'sdwanSite' && !ctx.sdwanSite) missing.push('sdwanSite'); + else if (req === 'sdwanData' && !ctx.sdwanData) missing.push('sdwanData'); } return missing; } diff --git a/tests/paloalto.client.test.js b/tests/paloalto.client.test.js new file mode 100644 index 0000000..51749b4 --- /dev/null +++ b/tests/paloalto.client.test.js @@ -0,0 +1,326 @@ +// tests/paloalto.client.test.js +// +// Coverage for integrations/paloalto/client.js — the dual-mode +// auth wrapper. Uses a fake HTTP server to exercise: +// - SASE OAuth token acquisition (form-encoded client_credentials +// against a fake auth URL) +// - Legacy CloudGenix login (JSON POST against /v2.0/api/login) +// - Mutex behaviour under concurrent callers +// - 401 → forced refresh + one-shot retry +// - PRISMA_AUTH_MODE=unknown throws a clear error +// +// Uses the same fake-server pattern as paloalto.sites.test.js. + +import test from 'node:test'; +import assert from 'node:assert/strict'; +import http from 'node:http'; + +import { paloAltoAxios, getPrismaToken, _resetPrismaAuthCache } from '../integrations/paloalto/client.js'; + +const FAKE_TOKEN_1 = 'token-round-1'; +const FAKE_TOKEN_2 = 'token-round-2'; + +async function makeFakeAuthServer(handlers = {}) { + const requests = []; + const server = http.createServer((req, res) => { + let body = ''; + req.on('data', (c) => (body += c)); + req.on('end', () => { + requests.push({ method: req.method, url: req.url, body, headers: req.headers }); + // Default SASE session-prime responder so tests that only care + // about the token / retry path don't need to wire this in. + // Individual tests can override by providing an explicit + // handler for `GET /sdwan/v2.1/api/profile`. + const explicitHandler = handlers[`${req.method} ${req.url}`]; + if ( + req.url === '/sdwan/v2.1/api/profile' && + req.method === 'GET' && + !explicitHandler + ) { + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ id: 'stub-profile' })); + return; + } + // Default OAuth token responder — same rationale as the prime + // default above. Overridable by explicit handler. + if ( + req.url === '/oauth2/access_token' && + req.method === 'POST' && + !explicitHandler + ) { + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ access_token: 'default-token', expires_in: 900 })); + return; + } + const handler = explicitHandler; + if (handler) { + const result = handler({ req, body }); + res.writeHead(result.status || 200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify(result.body || {})); + return; + } + res.writeHead(404); + res.end(); + }); + }); + await new Promise((r) => server.listen(0, '127.0.0.1', r)); + const { port } = server.address(); + return { + port, + baseUrl: `http://127.0.0.1:${port}`, + requests, + close: () => new Promise((r) => server.close(r)), + }; +} + +function clearAllEnv() { + delete process.env.PRISMA_AUTH_MODE; + delete process.env.PRISMA_SASE_BASE_URL; + delete process.env.PRISMA_LEGACY_BASE_URL; + delete process.env.PRISMA_AUTH_URL; + delete process.env.PRISMA_CLIENT_ID; + delete process.env.PRISMA_CLIENT_SECRET; + delete process.env.PRISMA_TSG_ID; + delete process.env.PRISMA_EMAIL; + delete process.env.PRISMA_PASSWORD; +} + +test('client: PRISMA_AUTH_MODE=unknown throws on first token request', async () => { + _resetPrismaAuthCache(); + clearAllEnv(); + process.env.PRISMA_AUTH_MODE = 'nonsense'; + try { + await assert.rejects(() => getPrismaToken(true), /PRISMA_AUTH_MODE/); + } finally { + clearAllEnv(); + _resetPrismaAuthCache(); + } +}); + +test('client: SASE mode fetches token via client_credentials + scopes to tsg_id', async () => { + _resetPrismaAuthCache(); + clearAllEnv(); + const fake = await makeFakeAuthServer({ + 'POST /oauth2/access_token': ({ body }) => { + assert.match(body, /grant_type=client_credentials/); + assert.match(body, /scope=tsg_id%3A/); + return { status: 200, body: { access_token: FAKE_TOKEN_1, expires_in: 900 } }; + }, + }); + process.env.PRISMA_AUTH_MODE = 'sase'; + process.env.PRISMA_AUTH_URL = `${fake.baseUrl}/oauth2/access_token`; + process.env.PRISMA_CLIENT_ID = 'id'; + process.env.PRISMA_CLIENT_SECRET = 'secret'; + process.env.PRISMA_TSG_ID = 'tsg'; + try { + const t = await getPrismaToken(true); + assert.equal(t, FAKE_TOKEN_1); + } finally { + await fake.close(); + clearAllEnv(); + _resetPrismaAuthCache(); + } +}); + +test('client: SASE mode surfaces missing envs with a clear error', async () => { + _resetPrismaAuthCache(); + clearAllEnv(); + process.env.PRISMA_AUTH_MODE = 'sase'; + try { + await assert.rejects(() => getPrismaToken(true), /PRISMA_CLIENT_ID/); + } finally { + clearAllEnv(); + _resetPrismaAuthCache(); + } +}); + +test('client: legacy mode POSTs email+password to /v2.0/api/login', async () => { + _resetPrismaAuthCache(); + clearAllEnv(); + const fake = await makeFakeAuthServer({ + 'POST /v2.0/api/login': ({ body }) => { + const parsed = JSON.parse(body); + assert.equal(parsed.email, 'a@b.com'); + assert.equal(parsed.password, 'secret'); + return { status: 200, body: { x_auth_token: 'legacy-token-xyz' } }; + }, + }); + process.env.PRISMA_AUTH_MODE = 'legacy'; + process.env.PRISMA_LEGACY_BASE_URL = fake.baseUrl; + process.env.PRISMA_EMAIL = 'a@b.com'; + process.env.PRISMA_PASSWORD = 'secret'; + try { + const t = await getPrismaToken(true); + assert.equal(t, 'legacy-token-xyz'); + } finally { + await fake.close(); + clearAllEnv(); + _resetPrismaAuthCache(); + } +}); + +test('client: legacy mode surfaces missing envs with a clear error', async () => { + _resetPrismaAuthCache(); + clearAllEnv(); + process.env.PRISMA_AUTH_MODE = 'legacy'; + try { + await assert.rejects(() => getPrismaToken(true), /PRISMA_EMAIL/); + } finally { + clearAllEnv(); + _resetPrismaAuthCache(); + } +}); + +test('client: concurrent callers coalesce onto a single token refresh', async () => { + _resetPrismaAuthCache(); + clearAllEnv(); + let hits = 0; + const fake = await makeFakeAuthServer({ + 'POST /oauth2/access_token': () => { + hits += 1; + return { status: 200, body: { access_token: `token-${hits}`, expires_in: 900 } }; + }, + }); + process.env.PRISMA_AUTH_MODE = 'sase'; + process.env.PRISMA_AUTH_URL = `${fake.baseUrl}/oauth2/access_token`; + process.env.PRISMA_CLIENT_ID = 'id'; + process.env.PRISMA_CLIENT_SECRET = 'secret'; + process.env.PRISMA_TSG_ID = 'tsg'; + try { + const tokens = await Promise.all([ + getPrismaToken(true), + getPrismaToken(false), + getPrismaToken(false), + getPrismaToken(false), + ]); + assert.equal(hits, 1, 'mutex should coalesce concurrent refresh requests'); + assert.equal(tokens[0], 'token-1'); + // Subsequent callers should get the same cached token as the + // first (mutex holds them until it's cached). + assert.equal(tokens[1], tokens[0]); + assert.equal(tokens[2], tokens[0]); + assert.equal(tokens[3], tokens[0]); + } finally { + await fake.close(); + clearAllEnv(); + _resetPrismaAuthCache(); + } +}); + +test('client: SASE mode primes session with GET /sdwan/v2.1/api/profile before the first SD-WAN call', async () => { + _resetPrismaAuthCache(); + clearAllEnv(); + let profileHits = 0; + let apiHits = 0; + const fake = await makeFakeAuthServer({ + 'GET /sdwan/v2.1/api/profile': ({ req }) => { + profileHits += 1; + assert.match(req.headers.authorization || '', /^Bearer /); + return { status: 200, body: { id: 'stub-profile', tenant_id: 't' } }; + }, + 'GET /sdwan/v4.13/api/sites': () => { + apiHits += 1; + return { status: 200, body: { items: [] } }; + }, + }); + process.env.PRISMA_AUTH_MODE = 'sase'; + process.env.PRISMA_SASE_BASE_URL = fake.baseUrl; + process.env.PRISMA_AUTH_URL = `${fake.baseUrl}/oauth2/access_token`; + process.env.PRISMA_CLIENT_ID = 'id'; + process.env.PRISMA_CLIENT_SECRET = 'secret'; + process.env.PRISMA_TSG_ID = 'tsg'; + try { + await paloAltoAxios.get('/sdwan/v4.13/api/sites'); + await paloAltoAxios.get('/sdwan/v4.13/api/sites'); + await paloAltoAxios.get('/sdwan/v4.13/api/sites'); + assert.equal(profileHits, 1, 'priming call should fire exactly once per token'); + assert.equal(apiHits, 3, 'subsequent SD-WAN calls should all succeed'); + // Ordering assertion: /profile happened before the first SD-WAN call. + const seqUrls = fake.requests.map((r) => r.url).filter((u) => u.startsWith('/sdwan/')); + assert.equal(seqUrls[0], '/sdwan/v2.1/api/profile'); + } finally { + await fake.close(); + clearAllEnv(); + _resetPrismaAuthCache(); + } +}); + +test('client: SASE priming re-runs after a forced token refresh', async () => { + _resetPrismaAuthCache(); + clearAllEnv(); + let profileHits = 0; + const fake = await makeFakeAuthServer({ + 'GET /sdwan/v2.1/api/profile': () => { + profileHits += 1; + return { status: 200, body: { id: 'stub-profile' } }; + }, + 'GET /sdwan/v4.13/api/sites': () => ({ status: 200, body: { items: [] } }), + }); + process.env.PRISMA_AUTH_MODE = 'sase'; + process.env.PRISMA_SASE_BASE_URL = fake.baseUrl; + process.env.PRISMA_AUTH_URL = `${fake.baseUrl}/oauth2/access_token`; + process.env.PRISMA_CLIENT_ID = 'id'; + process.env.PRISMA_CLIENT_SECRET = 'secret'; + process.env.PRISMA_TSG_ID = 'tsg'; + try { + await paloAltoAxios.get('/sdwan/v4.13/api/sites'); + assert.equal(profileHits, 1); + // Force a token refresh — priming should re-run on next call. + await getPrismaToken(true); + await paloAltoAxios.get('/sdwan/v4.13/api/sites'); + assert.equal(profileHits, 2, 'new token → re-prime'); + } finally { + await fake.close(); + clearAllEnv(); + _resetPrismaAuthCache(); + } +}); + +test('client: axios instance retries once after 401 with forced refresh', async () => { + _resetPrismaAuthCache(); + clearAllEnv(); + let tokenCallCount = 0; + let apiCallCount = 0; + const fake = await makeFakeAuthServer({ + 'POST /oauth2/access_token': () => { + tokenCallCount += 1; + return { + status: 200, + body: { + access_token: tokenCallCount === 1 ? FAKE_TOKEN_1 : FAKE_TOKEN_2, + expires_in: 900, + }, + }; + }, + 'GET /some/api/endpoint': ({ req }) => { + apiCallCount += 1; + // First call → 401 to trigger refresh. Second call must + // present the new token to succeed. + if (apiCallCount === 1) { + return { status: 401, body: { error: 'expired' } }; + } + const auth = req.headers.authorization || ''; + if (auth === `Bearer ${FAKE_TOKEN_2}`) { + return { status: 200, body: { ok: true } }; + } + return { status: 401, body: { error: 'still bad' } }; + }, + }); + process.env.PRISMA_AUTH_MODE = 'sase'; + process.env.PRISMA_SASE_BASE_URL = fake.baseUrl; + process.env.PRISMA_AUTH_URL = `${fake.baseUrl}/oauth2/access_token`; + process.env.PRISMA_CLIENT_ID = 'id'; + process.env.PRISMA_CLIENT_SECRET = 'secret'; + process.env.PRISMA_TSG_ID = 'tsg'; + try { + const res = await paloAltoAxios.get('/some/api/endpoint'); + assert.equal(res.status, 200); + assert.equal(res.data.ok, true); + assert.equal(tokenCallCount, 2, 'token refresh happened after 401'); + assert.equal(apiCallCount, 2, 'API call retried once'); + } finally { + await fake.close(); + clearAllEnv(); + _resetPrismaAuthCache(); + } +}); diff --git a/tests/paloalto.metrics.test.js b/tests/paloalto.metrics.test.js new file mode 100644 index 0000000..a0aa6ac --- /dev/null +++ b/tests/paloalto.metrics.test.js @@ -0,0 +1,331 @@ +// tests/paloalto.metrics.test.js +// +// Regression guards for the Prisma SD-WAN metric-body shapes. Every +// bug in this file was found the hard way — live tenant returned +// SCHEMA_CHECK_FAIL 400s that took a round trip to pin down. Keep +// these tests thin, focused, and worded so a future refactor can +// see exactly WHICH schema constraint the assertion is protecting. + +import test from 'node:test'; +import assert from 'node:assert/strict'; +import http from 'node:http'; + +import { + getHealthscore, + getLqmMetric, + getAlarms, +} from '../integrations/paloalto/metrics.js'; +import { _resetPrismaAuthCache } from '../integrations/paloalto/client.js'; + +async function makeFakePrisma(routes) { + const server = http.createServer((req, res) => { + let body = ''; + req.on('data', (c) => (body += c)); + req.on('end', () => { + if (req.url === '/oauth2/access_token') { + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ access_token: 't', expires_in: 900 })); + return; + } + if (req.url === '/sdwan/v2.1/api/profile' && req.method === 'GET') { + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ id: 'stub-profile' })); + return; + } + const h = routes[`${req.method} ${req.url}`]; + if (h) { + const parsed = body ? JSON.parse(body) : null; + const r = h({ req, body: parsed }); + res.writeHead(r.status || 200, { 'Content-Type': 'application/json', ...(r.headers || {}) }); + res.end(JSON.stringify(r.body || {})); + return; + } + res.writeHead(404); + res.end(); + }); + }); + await new Promise((r) => server.listen(0, '127.0.0.1', r)); + const { port } = server.address(); + return { baseUrl: `http://127.0.0.1:${port}`, close: () => new Promise((r) => server.close(r)) }; +} + +function setSaseEnv(baseUrl) { + process.env.PRISMA_AUTH_MODE = 'sase'; + process.env.PRISMA_SASE_BASE_URL = baseUrl; + process.env.PRISMA_AUTH_URL = `${baseUrl}/oauth2/access_token`; + process.env.PRISMA_CLIENT_ID = 'id'; + process.env.PRISMA_CLIENT_SECRET = 'secret'; + process.env.PRISMA_TSG_ID = 'tsg'; +} +function clearEnv() { + ['PRISMA_AUTH_MODE','PRISMA_SASE_BASE_URL','PRISMA_AUTH_URL', + 'PRISMA_CLIENT_ID','PRISMA_CLIENT_SECRET','PRISMA_TSG_ID', + 'PRISMA_LEGACY_BASE_URL','PRISMA_EMAIL','PRISMA_PASSWORD'] + .forEach((k) => delete process.env[k]); +} + +// ─── Body-shape regressions ───────────────────────────────────────── + +test('getHealthscore: uses v2.6 unified metrics endpoint (v2.0 aiops/health is a dead end on this tenant)', async () => { + _resetPrismaAuthCache(); + const validIntervals = new Set(['10sec', '1min', '5min', '1hour', '1day']); + let seenBody = null; + let seenUrl = null; + const fake = await makeFakePrisma({ + 'POST /sdwan/monitor/v2.6/api/monitor/metrics': ({ req, body }) => { + seenBody = body; + seenUrl = req.url; + return { body: { metrics: [] } }; + }, + // Trip-wire on the old v2.0 aiops/health endpoint. Fails loudly + // if a future refactor accidentally regresses to it. + 'POST /sdwan/monitor/v2.0/api/monitor/aiops/health': () => { + throw new Error('REGRESSION: healthscore should hit v2.6 /monitor/metrics, not v2.0 /aiops/health'); + }, + }); + setSaseEnv(fake.baseUrl); + try { + await getHealthscore('site-A'); + // v2.6 endpoint is the ONLY healthscore endpoint that returns 200 + // on the observed tenant (verified 2026-07-09 via prismaProbe + // try-shapes; both v2.0 aiops/health and v2.1 aggregates dead-end + // on tenant-enforced schema errors). + assert.equal(seenUrl, '/sdwan/monitor/v2.6/api/monitor/metrics'); + assert.deepEqual(seenBody.filter, { site: ['site-A'] }, + 'v2.6 accepts filter.site as an ARRAY (unlike v2.0)'); + assert.deepEqual(seenBody.view, {}, + 'v2.6 accepts view as an empty object (unlike v2.0 aiops/health which wants a string enum)'); + assert.equal(seenBody.metrics[0].name, 'Healthscore'); + assert.deepEqual(seenBody.metrics[0].statistics, ['max']); + assert.equal(seenBody.metrics[0].unit, 'gauge'); + assert.equal( + Object.prototype.hasOwnProperty.call(seenBody, 'end_time'), false, + 'no end_time — endpoint interprets window as [start_time, now)', + ); + assert.ok(validIntervals.has(seenBody.interval), + `interval must be one of ${[...validIntervals].join(', ')} — got "${seenBody.interval}"`); + } finally { + await fake.close(); _resetPrismaAuthCache(); clearEnv(); + } +}); + +test('getLqmMetric: uses dedicated lqm_point_metrics endpoint (not sys_point_metrics)', async () => { + _resetPrismaAuthCache(); + let seenBody = null; + let seenUrl = null; + const fake = await makeFakePrisma({ + 'POST /sdwan/monitor/v2.0/api/monitor/lqm_point_metrics': ({ req, body }) => { + seenBody = body; + seenUrl = req.url; + return { body: { metrics: [] } }; + }, + // Regression trip-wire: sys_point_metrics is the CPU/Memory/Disk + // endpoint and rejects filter.wan_interfaces. If we regress to it, + // fail the test with a specific message instead of a generic 404. + 'POST /sdwan/monitor/v2.0/api/monitor/sys_point_metrics': () => { + throw new Error('REGRESSION: getLqmMetric should use lqm_point_metrics, not sys_point_metrics'); + }, + }); + setSaseEnv(fake.baseUrl); + try { + await getLqmMetric('site-A', ['wi-1', 'wi-2'], 'latency'); + assert.equal(seenUrl, '/sdwan/monitor/v2.0/api/monitor/lqm_point_metrics'); + // Live 400 regressions + LIVEcommunity #1235108 working example: + // - `end_time` is REJECTED ("not defined in the schema"). The + // endpoint interprets the window as [start_time, now). + // - filter.site must be an ARRAY on lqm_point_metrics — 400 was + // "$.filter.site: string found, array expected". OPPOSITE of + // sys_point_metrics, which wants a string. + // - filter.path is the KEY THAT WORKS. Both `wan_interfaces` + // and `waninterface` were rejected as "not defined in the + // schema". The LIVEcommunity example for LqmLatency uses: + // filter: { site: [id], path: [wi_id] } + assert.equal( + Object.prototype.hasOwnProperty.call(seenBody, 'end_time'), false, + 'lqm_point_metrics rejects `end_time` — do not send', + ); + assert.ok(Array.isArray(seenBody.filter.site), + 'lqm_point_metrics requires filter.site as an ARRAY (opposite of sys_point_metrics)'); + assert.deepEqual(seenBody.filter.site, ['site-A']); + assert.deepEqual(seenBody.filter.path, ['wi-1', 'wi-2'], + 'filter key is `path` — NOT wan_interfaces (rejected), NOT waninterface (sys_point_metrics variant)'); + assert.equal( + Object.prototype.hasOwnProperty.call(seenBody.filter, 'wan_interfaces'), + false, + 'do NOT send `wan_interfaces` — rejected as "not defined in the schema"', + ); + assert.equal( + Object.prototype.hasOwnProperty.call(seenBody.filter, 'waninterface'), + false, + 'do NOT send `waninterface` — that key belongs to sys_point_metrics', + ); + assert.equal( + Object.prototype.hasOwnProperty.call(seenBody.filter, 'elements'), + false, + 'do NOT send `elements` — that was a sys_point_metrics quirk', + ); + assert.equal(seenBody.metrics[0].name, 'LqmLatencyPointMetric'); + assert.equal(seenBody.metrics[0].unit, 'milliseconds'); + } finally { + await fake.close(); _resetPrismaAuthCache(); clearEnv(); + } +}); + +test('getLqmMetric: correct PointMetric name + unit per key (jitter/loss/mos)', async () => { + _resetPrismaAuthCache(); + const seen = []; + const fake = await makeFakePrisma({ + 'POST /sdwan/monitor/v2.0/api/monitor/lqm_point_metrics': ({ body }) => { + seen.push(body.metrics[0]); + return { body: { metrics: [] } }; + }, + }); + setSaseEnv(fake.baseUrl); + try { + await getLqmMetric('site-A', ['wi-1'], 'jitter'); + await getLqmMetric('site-A', ['wi-1'], 'loss'); + await getLqmMetric('site-A', ['wi-1'], 'mos'); + assert.deepEqual(seen.map((m) => [m.name, m.unit]), [ + // Regression: units are CASE-SENSITIVE per Prisma's schema + // validator. `Percentage` (capital P) was rejected on this + // tenant with 400 METRIC_UNIT_NOT_SUPPORTED — the correct + // spelling is lowercase `percentage`. Verified 2026-07-09 via + // `try-shapes lqm-loss`. + ['LqmJitterPointMetric', 'milliseconds'], + ['LqmPktLossPointMetric', 'percentage'], + ['LqmMosPointMetric', 'count'], + ]); + } finally { + await fake.close(); _resetPrismaAuthCache(); clearEnv(); + } +}); + +test('getLqmMetric: empty waninterfaceIds → skip (no HTTP call)', async () => { + _resetPrismaAuthCache(); + let called = false; + const fake = await makeFakePrisma({ + 'POST /sdwan/monitor/v2.0/api/monitor/lqm_point_metrics': () => { + called = true; + return { body: { metrics: [] } }; + }, + }); + setSaseEnv(fake.baseUrl); + try { + assert.equal(await getLqmMetric('site-A', [], 'latency'), null); + assert.equal(called, false); + } finally { + await fake.close(); _resetPrismaAuthCache(); clearEnv(); + } +}); + +// ─── 429 retry ────────────────────────────────────────────────────── + +test('client: 429 → retry with exponential backoff, then succeed on second attempt', async () => { + _resetPrismaAuthCache(); + let attempt = 0; + const startTs = Date.now(); + const fake = await makeFakePrisma({ + 'POST /sdwan/monitor/v2.6/api/monitor/metrics': () => { + attempt += 1; + if (attempt === 1) { + return { status: 429, body: { error: 'too many' } }; + } + return { body: { metrics: [{ name: 'Healthscore', series: [{ data: [{ value: 95 }] }] }] } }; + }, + }); + setSaseEnv(fake.baseUrl); + try { + const resp = await getHealthscore('site-A'); + const elapsed = Date.now() - startTs; + assert.equal(attempt, 2, 'should have retried exactly once'); + assert.ok(resp, 'second attempt should return the payload'); + assert.equal(resp.metrics[0].name, 'Healthscore'); + assert.ok(elapsed >= 1500, `should have waited ~1.5s (base backoff) — waited ${elapsed}ms`); + } finally { + await fake.close(); _resetPrismaAuthCache(); clearEnv(); + } +}); + +test('client: 429 → honors Retry-After header when present', async () => { + _resetPrismaAuthCache(); + let attempt = 0; + const startTs = Date.now(); + const fake = await makeFakePrisma({ + 'POST /sdwan/monitor/v2.6/api/monitor/metrics': () => { + attempt += 1; + if (attempt === 1) { + // Retry-After of 2 seconds — should be honored over the + // client's default 1.5s exponential backoff. + return { status: 429, headers: { 'Retry-After': '2' }, body: {} }; + } + return { body: { metrics: [] } }; + }, + }); + setSaseEnv(fake.baseUrl); + try { + await getHealthscore('site-A'); + const elapsed = Date.now() - startTs; + assert.ok(elapsed >= 2000, `should have waited ~2s per Retry-After — waited ${elapsed}ms`); + assert.equal(attempt, 2); + } finally { + await fake.close(); _resetPrismaAuthCache(); clearEnv(); + } +}); + +test('client: 429 → gives up after MAX_429_RETRIES and returns null', async () => { + _resetPrismaAuthCache(); + let attempt = 0; + const fake = await makeFakePrisma({ + 'POST /sdwan/monitor/v2.6/api/monitor/metrics': () => { + attempt += 1; + return { status: 429, body: {} }; + }, + }); + setSaseEnv(fake.baseUrl); + try { + const resp = await getHealthscore('site-A'); + assert.equal(resp, null, 'metric wrapper absorbs the exhausted-retry error'); + // 1 original + 2 retries = 3 total attempts. + assert.equal(attempt, 3, 'should attempt exactly MAX_429_RETRIES + 1 times'); + } finally { + await fake.close(); _resetPrismaAuthCache(); clearEnv(); + } +}); + +// ─── Alarms body shape (events/query endpoint) ────────────────────── + +test('getAlarms: uses events/query endpoint (not /monitor/alarms which 404s)', async () => { + _resetPrismaAuthCache(); + let seenBody = null; + let seenUrl = null; + const fake = await makeFakePrisma({ + 'POST /sdwan/v3.7/api/events/query': ({ req, body }) => { + seenBody = body; + seenUrl = req.url; + return { body: { items: [] } }; + }, + // Regression trip-wire: if we accidentally regress to the old + // /monitor/alarms path, this handler catches it and fails the + // test with a specific message instead of a generic 404. + 'POST /sdwan/monitor/v2.0/api/monitor/alarms': () => { + throw new Error('REGRESSION: getAlarms should use /events/query, not /monitor/alarms'); + }, + }); + setSaseEnv(fake.baseUrl); + try { + const resp = await getAlarms('site-A'); + assert.ok(resp, 'events/query response returned'); + assert.equal(seenUrl, '/sdwan/v3.7/api/events/query'); + assert.deepEqual(seenBody.query.site, ['site-A']); + assert.deepEqual(seenBody.query.type, ['alarm'], + 'must filter for type=alarm — informational events would inflate the count'); + assert.deepEqual(seenBody.severity, ['critical', 'major', 'minor']); + assert.ok(seenBody.limit && typeof seenBody.limit === 'object', + 'limit is an OBJECT on events/query (count + sort_on + sort_order), not an int'); + assert.equal(seenBody.limit.sort_on, 'time'); + assert.equal(seenBody.limit.sort_order, 'descending'); + assert.ok(seenBody.start_time, 'start_time populated'); + } finally { + await fake.close(); _resetPrismaAuthCache(); clearEnv(); + } +}); diff --git a/tests/paloalto.sites.test.js b/tests/paloalto.sites.test.js new file mode 100644 index 0000000..a8c318d --- /dev/null +++ b/tests/paloalto.sites.test.js @@ -0,0 +1,666 @@ +// tests/paloalto.sites.test.js +// +// Unit + integration coverage for integrations/paloalto/sites.js. +// +// The `siteNameForStore()` function is pure — most cases are covered +// by inline assertions. The `findSdwanSiteForStore()` + +// `getElementsForSite()` functions hit HTTP; we stand up a tiny +// fake Prisma server on an ephemeral port and point the module at +// it via PRISMA_SASE_BASE_URL. Same pattern as dectRelayHub.test.js. +// +// Env cleanup runs after every test that touches process.env so +// one failing test can't poison the rest of the file. + +import test from 'node:test'; +import assert from 'node:assert/strict'; +import http from 'node:http'; + +import { + siteNameForStore, + findSdwanSiteForStore, + getAllSites, + getElementsForSite, + getWanInterfacesForSite, + _resetSitesCache, +} from '../integrations/paloalto/sites.js'; +import { _resetPrismaAuthCache } from '../integrations/paloalto/client.js'; + +const FAKE_TOKEN = 'test-token-abc123'; + +// ─── Fake Prisma server ───────────────────────────────────────────── +// +// Handles just enough of the two endpoints the sites module talks +// to. Each test passes a `handlers` map so it can control what +// each endpoint returns — a missing handler responds 404. All +// requests get authenticated against FAKE_TOKEN so the client's +// interceptor + refresh flow gets exercised. + +async function makeFakePrisma(handlers = {}) { + const requests = []; + const server = http.createServer((req, res) => { + let body = ''; + req.on('data', (c) => (body += c)); + req.on('end', () => { + requests.push({ method: req.method, url: req.url, body, headers: req.headers }); + + // Auth token endpoint (SASE). The auth URL is the whole URL + // (not a path on baseURL) — but we override it too so + // the auth POST comes here. + if (req.url === '/oauth2/access_token' && req.method === 'POST') { + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ access_token: FAKE_TOKEN, expires_in: 900 })); + return; + } + + // Mandatory SASE unified SD-WAN session priming call. Every + // token acquisition should be followed by exactly one hit + // against this URL — we respond with a stub profile. + if (req.url === '/sdwan/v2.1/api/profile' && req.method === 'GET') { + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ id: 'stub-profile', tenant_id: 'stub-tenant' })); + return; + } + + // Match on method + path (ignoring query string) so tests + // don't have to encode every ?limit=1000&cursor=... permutation. + // Query params are still preserved on `req.query` and the raw + // `req.url` for handlers that need them. + const pathOnly = (req.url || '').split('?')[0]; + const rawQuery = (req.url || '').includes('?') ? req.url.split('?')[1] : ''; + const query = Object.fromEntries(new URLSearchParams(rawQuery)); + const handler = + handlers[`${req.method} ${req.url}`] || // exact match wins + handlers[`${req.method} ${pathOnly}`]; // path-only fallback + if (handler) { + const parsed = body ? JSON.parse(body) : null; + const result = handler({ req, body: parsed, query }); + res.writeHead(result.status || 200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify(result.body || {})); + return; + } + res.writeHead(404); + res.end(); + }); + }); + await new Promise((r) => server.listen(0, '127.0.0.1', r)); + const { port } = server.address(); + return { + port, + baseUrl: `http://127.0.0.1:${port}`, + requests, + close: () => new Promise((r) => server.close(r)), + }; +} + +function setSaseEnv(baseUrl) { + process.env.PRISMA_AUTH_MODE = 'sase'; + process.env.PRISMA_SASE_BASE_URL = baseUrl; + process.env.PRISMA_AUTH_URL = `${baseUrl}/oauth2/access_token`; + process.env.PRISMA_CLIENT_ID = 'test-client'; + process.env.PRISMA_CLIENT_SECRET = 'test-secret'; + process.env.PRISMA_TSG_ID = 'test-tsg'; +} + +function clearPrismaEnv() { + delete process.env.PRISMA_AUTH_MODE; + delete process.env.PRISMA_SASE_BASE_URL; + delete process.env.PRISMA_AUTH_URL; + delete process.env.PRISMA_CLIENT_ID; + delete process.env.PRISMA_CLIENT_SECRET; + delete process.env.PRISMA_TSG_ID; + delete process.env.PRISMA_LEGACY_BASE_URL; + delete process.env.PRISMA_EMAIL; + delete process.env.PRISMA_PASSWORD; +} + +// ─── siteNameForStore (pure) ──────────────────────────────────────── + +test('siteNameForStore: pads 3-digit store to 5 with CG prefix', () => { + assert.equal(siteNameForStore(782), 'CG00782'); + assert.equal(siteNameForStore('782'), 'CG00782'); +}); + +test('siteNameForStore: pads 5-digit store correctly', () => { + assert.equal(siteNameForStore(2477), 'CG02477'); + assert.equal(siteNameForStore(305), 'CG00305'); +}); + +test('siteNameForStore: handles 4-digit stores', () => { + assert.equal(siteNameForStore(1234), 'CG01234'); +}); + +test('siteNameForStore: strips non-digits before padding', () => { + assert.equal(siteNameForStore('store-782'), 'CG00782'); +}); + +test('siteNameForStore: throws on empty / non-numeric input', () => { + assert.throws(() => siteNameForStore(''), /no digits/i); + assert.throws(() => siteNameForStore('abc'), /no digits/i); + assert.throws(() => siteNameForStore(null), /no digits/i); +}); + +// ─── findSdwanSiteForStore + cache ────────────────────────────────── + +test('findSdwanSiteForStore: exact match on CG00782', async () => { + _resetSitesCache(); + _resetPrismaAuthCache(); + const fake = await makeFakePrisma({ + 'GET /sdwan/v4.13/api/sites': () => ({ + status: 200, + body: { + items: [ + { id: 'site-1', name: 'CG00305', description: 'store 305' }, + { id: 'site-2', name: 'CG00782', description: 'store 782' }, + { id: 'site-3', name: 'CG02477', description: 'store 2477' }, + ], + }, + }), + }); + setSaseEnv(fake.baseUrl); + try { + const site = await findSdwanSiteForStore(782); + assert.ok(site, 'expected site to resolve'); + assert.equal(site.name, 'CG00782'); + assert.equal(site.id, 'site-2'); + } finally { + await fake.close(); + _resetSitesCache(); + _resetPrismaAuthCache(); + clearPrismaEnv(); + } +}); + +test('findSdwanSiteForStore: null when no match', async () => { + _resetSitesCache(); + _resetPrismaAuthCache(); + const fake = await makeFakePrisma({ + 'GET /sdwan/v4.13/api/sites': () => ({ + status: 200, + body: { items: [{ id: 'site-x', name: 'CG99999', description: '' }] }, + }), + }); + setSaseEnv(fake.baseUrl); + try { + const site = await findSdwanSiteForStore(782); + assert.equal(site, null); + } finally { + await fake.close(); + _resetSitesCache(); + _resetPrismaAuthCache(); + clearPrismaEnv(); + } +}); + +test('getAllSites: caches — second call does not re-fetch', async () => { + _resetSitesCache(); + _resetPrismaAuthCache(); + let fetchCount = 0; + const fake = await makeFakePrisma({ + 'GET /sdwan/v4.13/api/sites': () => { + fetchCount += 1; + return { status: 200, body: { items: [{ id: 's', name: 'CG00782' }] } }; + }, + }); + setSaseEnv(fake.baseUrl); + try { + await getAllSites(); + await getAllSites(); + await getAllSites(); + assert.equal(fetchCount, 1, 'only one refresh expected within TTL'); + } finally { + await fake.close(); + _resetSitesCache(); + _resetPrismaAuthCache(); + clearPrismaEnv(); + } +}); + +test('getAllSites: uses GET (not POST) so no request body → no schema games', async () => { + // Regression guard: the observed SASE tenant rejects POST + // /sites/query with a moving target of body-schema errors + // (getDeleted expected bool, total_count expected long, etc.) + // even when we send those fields correctly. The tenant's SASE + // proxy appears to auto-inject fields with `{}` defaults into + // the body before schema validation, making POST unusable. + // GET has no body, so no auto-injection, so no drift. + _resetSitesCache(); + _resetPrismaAuthCache(); + const seen = { method: null, path: null, query: null, bodyLen: 0 }; + const fake = await makeFakePrisma({ + 'GET /sdwan/v4.13/api/sites': ({ req, body, query }) => { + seen.method = req.method; + seen.path = (req.url || '').split('?')[0]; + seen.query = query; + seen.bodyLen = body ? JSON.stringify(body).length : 0; + return { status: 200, body: { items: [{ id: 's', name: 'CG00782' }] } }; + }, + }); + setSaseEnv(fake.baseUrl); + try { + await getAllSites(); + assert.equal(seen.method, 'GET', 'MUST be GET, not POST'); + assert.equal(seen.path, '/sdwan/v4.13/api/sites'); + assert.equal(seen.bodyLen, 0, 'no request body'); + assert.equal(seen.query.limit, '1000', 'first-page limit maximised'); + } finally { + await fake.close(); + _resetSitesCache(); + _resetPrismaAuthCache(); + clearPrismaEnv(); + } +}); + +test('getAllSites: forceRefresh bypasses cache', async () => { + _resetSitesCache(); + _resetPrismaAuthCache(); + let fetchCount = 0; + const fake = await makeFakePrisma({ + 'GET /sdwan/v4.13/api/sites': () => { + fetchCount += 1; + return { status: 200, body: { items: [{ id: 's', name: 'CG00782' }] } }; + }, + }); + setSaseEnv(fake.baseUrl); + try { + await getAllSites(); + await getAllSites(true); + assert.equal(fetchCount, 2); + } finally { + await fake.close(); + _resetSitesCache(); + _resetPrismaAuthCache(); + clearPrismaEnv(); + } +}); + +test('getAllSites: follows next_query cursor across multiple pages', async () => { + _resetSitesCache(); + _resetPrismaAuthCache(); + let pageCallCount = 0; + const pages = [ + // Page 1: full page (1000 items) + a cursor that gets echoed + // back verbatim as the next request body. + { + items: Array.from({ length: 1000 }, (_, i) => ({ id: `s${i}`, name: `CG${String(i).padStart(5, '0')}` })), + next_query: { limit: 1000, getDeleted: false, cursor: 'page-2' }, + }, + // Page 2: full page + another cursor. + { + items: Array.from({ length: 1000 }, (_, i) => ({ id: `s${1000 + i}`, name: `CG${String(1000 + i).padStart(5, '0')}` })), + next_query: { limit: 1000, getDeleted: false, cursor: 'page-3' }, + }, + // Page 3: short page → terminates the loop by "fewer than limit". + { + items: Array.from({ length: 234 }, (_, i) => ({ id: `s${2000 + i}`, name: `CG${String(2000 + i).padStart(5, '0')}` })), + next_query: null, + }, + ]; + const fake = await makeFakePrisma({ + 'GET /sdwan/v4.13/api/sites': () => { + const page = pages[pageCallCount] || { items: [], next_query: null }; + pageCallCount += 1; + return { status: 200, body: page }; + }, + }); + setSaseEnv(fake.baseUrl); + try { + const sites = await getAllSites(); + assert.equal(pageCallCount, 3, 'should fetch all three pages'); + assert.equal(sites.length, 1000 + 1000 + 234, 'should union every page'); + assert.equal(sites[0].name, 'CG00000'); + assert.equal(sites.at(-1).name, 'CG02233'); + } finally { + await fake.close(); + _resetSitesCache(); + _resetPrismaAuthCache(); + clearPrismaEnv(); + } +}); + +test('getAllSites: keeps paginating when Prisma caps pages below the requested limit', async () => { + // Regression guard for a live bug: Prisma silently caps some + // tenants at ~200 rows per page regardless of the requested + // `limit: 1000`. The original short-page termination heuristic + // cut the sweep off after the first page, hiding 800+ sites from + // the store-lookup resolver. `next_query` alone must decide. + _resetSitesCache(); + _resetPrismaAuthCache(); + let pageCallCount = 0; + const pages = [ + { + // Page 1: 200 items (capped, WELL below the requested 1000) + // BUT a non-empty next_query says there's more. + items: Array.from({ length: 200 }, (_, i) => ({ id: `s${i}`, name: `CG${String(i).padStart(5, '0')}` })), + next_query: { cursor: 'p2', limit: 1000 }, + }, + { + // Page 2: another capped 200 + more. + items: Array.from({ length: 200 }, (_, i) => ({ id: `s${200 + i}`, name: `CG${String(200 + i).padStart(5, '0')}` })), + next_query: { cursor: 'p3', limit: 1000 }, + }, + { + // Page 3: final short page + empty cursor. + items: Array.from({ length: 42 }, (_, i) => ({ id: `s${400 + i}`, name: `CG${String(400 + i).padStart(5, '0')}` })), + next_query: null, + }, + ]; + const fake = await makeFakePrisma({ + 'GET /sdwan/v4.13/api/sites': () => { + const page = pages[pageCallCount] || { items: [], next_query: null }; + pageCallCount += 1; + return { status: 200, body: page }; + }, + }); + setSaseEnv(fake.baseUrl); + try { + const sites = await getAllSites(); + assert.equal(pageCallCount, 3, 'must not short-circuit on the capped first page'); + assert.equal(sites.length, 442, 'all 3 pages unioned'); + assert.equal(sites.at(-1).name, 'CG00441'); + } finally { + await fake.close(); + _resetSitesCache(); + _resetPrismaAuthCache(); + clearPrismaEnv(); + } +}); + +test('getAllSites: terminates on empty next_query even if page is full-sized', async () => { + _resetSitesCache(); + _resetPrismaAuthCache(); + let pageCallCount = 0; + const fake = await makeFakePrisma({ + 'GET /sdwan/v4.13/api/sites': () => { + pageCallCount += 1; + // Full page (== limit) but next_query is an empty object → + // terminate. Guards against a Prisma quirk where the last + // page happens to be exactly the page-size boundary. + return { + status: 200, + body: { + items: Array.from({ length: 1000 }, (_, i) => ({ id: `s${i}`, name: `CG${String(i).padStart(5, '0')}` })), + next_query: {}, + }, + }; + }, + }); + setSaseEnv(fake.baseUrl); + try { + const sites = await getAllSites(); + assert.equal(pageCallCount, 1); + assert.equal(sites.length, 1000); + } finally { + await fake.close(); + _resetSitesCache(); + _resetPrismaAuthCache(); + clearPrismaEnv(); + } +}); + +test('getAllSites: findSdwanSiteForStore resolves a store on page 3 of a paginated tenant', async () => { + _resetSitesCache(); + _resetPrismaAuthCache(); + let pageCallCount = 0; + const pages = [ + { items: Array.from({ length: 1000 }, (_, i) => ({ id: `s${i}`, name: `CG${String(i).padStart(5, '0')}` })), + next_query: { cursor: 'p2' } }, + { items: Array.from({ length: 1000 }, (_, i) => ({ id: `s${1000 + i}`, name: `CG${String(1000 + i).padStart(5, '0')}` })), + next_query: { cursor: 'p3' } }, + // Store 2200 lives on page 3. + { items: [{ id: 'target', name: 'CG02200' }], + next_query: null }, + ]; + const fake = await makeFakePrisma({ + 'GET /sdwan/v4.13/api/sites': () => { + const page = pages[pageCallCount] || { items: [], next_query: null }; + pageCallCount += 1; + return { status: 200, body: page }; + }, + }); + setSaseEnv(fake.baseUrl); + try { + const site = await findSdwanSiteForStore(2200); + assert.ok(site, 'expected store 2200 to resolve'); + assert.equal(site.name, 'CG02200'); + assert.equal(site.id, 'target'); + } finally { + await fake.close(); + _resetSitesCache(); + _resetPrismaAuthCache(); + clearPrismaEnv(); + } +}); + +test('getAllSites: returns empty array on network error with no cache', async () => { + _resetSitesCache(); + _resetPrismaAuthCache(); + const fake = await makeFakePrisma({ + 'GET /sdwan/v4.13/api/sites': () => ({ + status: 500, + body: { error: 'boom' }, + }), + }); + setSaseEnv(fake.baseUrl); + try { + const sites = await getAllSites(); + assert.deepEqual(sites, []); + } finally { + await fake.close(); + _resetSitesCache(); + _resetPrismaAuthCache(); + clearPrismaEnv(); + } +}); + +test('getElementsForSite: single tenant-wide GET, indexed by site_id in-memory', async () => { + _resetSitesCache(); + _resetPrismaAuthCache(); + const seen = { count: 0, urls: [] }; + const fake = await makeFakePrisma({ + 'GET /sdwan/v3.1/api/elements': ({ req }) => { + seen.count += 1; + seen.urls.push(req.url); + return { + status: 200, + body: { + items: [ + { id: 'el-1', name: 'ION-1000-A', model_name: 'ION 1000', serial_number: 'SN1', connected: true, site_id: 'site-abc' }, + { id: 'el-2', name: 'ION-1000-B', model_name: 'ION 1000', serial_number: 'SN2', connected: false, site_id: 'site-abc' }, + { id: 'el-3', name: 'ION-3000-X', model_name: 'ION 3000', serial_number: 'SN3', connected: true, site_id: 'site-xyz' }, + ], + }, + }; + }, + }); + setSaseEnv(fake.baseUrl); + try { + const abc = await getElementsForSite('site-abc'); + assert.equal(abc.length, 2); + assert.equal(abc[0].id, 'el-1'); + assert.equal(abc[0].model, 'ION 1000'); + + const xyz = await getElementsForSite('site-xyz'); + assert.equal(xyz.length, 1); + assert.equal(xyz[0].id, 'el-3'); + + const unknown = await getElementsForSite('site-nope'); + assert.deepEqual(unknown, [], 'unknown site returns empty (no extra fetch)'); + + // Critical: only ONE network call across three site lookups. + assert.equal(seen.count, 1, 'tenant-wide cache serves subsequent site lookups'); + assert.equal(seen.urls[0], '/sdwan/v3.1/api/elements', 'no ?site_id= filter — full tenant fetch'); + } finally { + await fake.close(); + _resetSitesCache(); + _resetPrismaAuthCache(); + clearPrismaEnv(); + } +}); + +test('getElementsForSite: elements with no site_id are dropped', async () => { + _resetSitesCache(); + _resetPrismaAuthCache(); + const fake = await makeFakePrisma({ + 'GET /sdwan/v3.1/api/elements': () => ({ + status: 200, + body: { + items: [ + { id: 'el-1', name: 'A', connected: true, site_id: 'site-abc' }, + { id: 'el-orphan', name: 'unclaimed', connected: false, site_id: null }, + ], + }, + }), + }); + setSaseEnv(fake.baseUrl); + try { + const els = await getElementsForSite('site-abc'); + assert.equal(els.length, 1); + assert.equal(els[0].id, 'el-1'); + } finally { + await fake.close(); + _resetSitesCache(); + _resetPrismaAuthCache(); + clearPrismaEnv(); + } +}); + +test('getElementsForSite: caches tenant-wide (one fetch across many sites)', async () => { + _resetSitesCache(); + _resetPrismaAuthCache(); + let fetchCount = 0; + const fake = await makeFakePrisma({ + 'GET /sdwan/v3.1/api/elements': () => { + fetchCount += 1; + return { status: 200, body: { items: [ + { id: 'el-a', name: 'a', connected: true, site_id: 'site-1' }, + { id: 'el-b', name: 'b', connected: true, site_id: 'site-2' }, + { id: 'el-c', name: 'c', connected: true, site_id: 'site-3' }, + ] } }; + }, + }); + setSaseEnv(fake.baseUrl); + try { + await getElementsForSite('site-1'); + await getElementsForSite('site-2'); + await getElementsForSite('site-3'); + await getElementsForSite('site-1'); // re-hit, no extra fetch + assert.equal(fetchCount, 1); + } finally { + await fake.close(); + _resetSitesCache(); + _resetPrismaAuthCache(); + clearPrismaEnv(); + } +}); + +test('getWanInterfacesForSite: GETs per-site waninterfaces, normalises to rows with admin state', async () => { + _resetSitesCache(); + _resetPrismaAuthCache(); + const fake = await makeFakePrisma({ + 'GET /sdwan/v2.10/api/sites/site-abc/waninterfaces': () => ({ + status: 200, + body: { + items: [ + { id: 'wi-1', name: 'MPLS Circuit', admin_up: true, used_for: 'primary', wan_network_id: 'net-1', bw_config_mode: 'manual' }, + { id: 'wi-2', name: 'Broadband', admin_up: true, used_for: 'secondary', wan_network_id: 'net-2', bw_config_mode: 'manual' }, + { id: 'wi-3', name: 'LTE Backup', admin_up: false, used_for: 'backup', wan_network_id: 'net-3', bw_config_mode: 'manual' }, + ], + }, + }), + }); + setSaseEnv(fake.baseUrl); + try { + const wans = await getWanInterfacesForSite('site-abc'); + assert.equal(wans.length, 3); + assert.equal(wans[0].id, 'wi-1'); + assert.equal(wans[0].name, 'MPLS Circuit'); + assert.equal(wans[0].adminUp, true); + assert.equal(wans[0].usedFor, 'primary'); + assert.equal(wans[2].adminUp, false); + } finally { + await fake.close(); + _resetSitesCache(); + _resetPrismaAuthCache(); + clearPrismaEnv(); + } +}); + +test('getWanInterfacesForSite: missing admin_up field → adminUp:null (not false)', async () => { + // Live regression: some tenant schema variants don't return an + // `admin_up` field on the waninterface config. Old code did + // `w.admin_up === true`, which falsy-coerced missing → false and + // then rendered every circuit as ❌ DOWN — a misleading false + // positive. The parser must distinguish missing (unknown) from + // an explicit `false` value. + _resetSitesCache(); + _resetPrismaAuthCache(); + const fake = await makeFakePrisma({ + 'GET /sdwan/v2.10/api/sites/site-x/waninterfaces': () => ({ + status: 200, + body: { + items: [ + { id: 'wi-a', name: 'no admin_up field at all' }, + { id: 'wi-b', name: 'admin_up=null', admin_up: null }, + { id: 'wi-c', name: 'admin_up="true" string', admin_up: 'true' }, + { id: 'wi-d', name: 'admin_up=true bool', admin_up: true }, + { id: 'wi-e', name: 'admin_up=false bool', admin_up: false }, + ], + }, + }), + }); + setSaseEnv(fake.baseUrl); + try { + const wans = await getWanInterfacesForSite('site-x'); + assert.equal(wans[0].adminUp, null, 'missing field → null (unknown)'); + assert.equal(wans[1].adminUp, null, 'explicit null → null (unknown)'); + assert.equal(wans[2].adminUp, null, 'stringy "true" is not a boolean → null'); + assert.equal(wans[3].adminUp, true, 'explicit true stays true'); + assert.equal(wans[4].adminUp, false, 'explicit false stays false (admin-disabled)'); + } finally { + await fake.close(); + _resetSitesCache(); + _resetPrismaAuthCache(); + clearPrismaEnv(); + } +}); + +test('getWanInterfacesForSite: caches per-site', async () => { + _resetSitesCache(); + _resetPrismaAuthCache(); + let fetchCount = 0; + const fake = await makeFakePrisma({ + 'GET /sdwan/v2.10/api/sites/site-x/waninterfaces': () => { + fetchCount += 1; + return { status: 200, body: { items: [{ id: 'wi', name: 'x', admin_up: true }] } }; + }, + }); + setSaseEnv(fake.baseUrl); + try { + await getWanInterfacesForSite('site-x'); + await getWanInterfacesForSite('site-x'); + assert.equal(fetchCount, 1); + } finally { + await fake.close(); + _resetSitesCache(); + _resetPrismaAuthCache(); + clearPrismaEnv(); + } +}); + +test('getWanInterfacesForSite: returns empty list on 500 with no cache', async () => { + _resetSitesCache(); + _resetPrismaAuthCache(); + const fake = await makeFakePrisma({ + 'GET /sdwan/v2.10/api/sites/broken/waninterfaces': () => ({ status: 500, body: { error: 'nope' } }), + }); + setSaseEnv(fake.baseUrl); + try { + const wans = await getWanInterfacesForSite('broken'); + assert.deepEqual(wans, []); + } finally { + await fake.close(); + _resetSitesCache(); + _resetPrismaAuthCache(); + clearPrismaEnv(); + } +}); diff --git a/tests/renderers.wan.test.js b/tests/renderers.wan.test.js new file mode 100644 index 0000000..dc794eb --- /dev/null +++ b/tests/renderers.wan.test.js @@ -0,0 +1,170 @@ +// tests/renderers.wan.test.js +// +// Pure-function coverage for services/renderers/wanDiagnosticsRenderer.js. +// Same style as tests/renderers.test.js — asserts on the returned +// markdown string. No mocking, no HTTP. + +import test from 'node:test'; +import assert from 'node:assert/strict'; + +import { renderWanDiagnosticsMarkdown } from '../services/renderers/wanDiagnosticsRenderer.js'; + +function baseData(overrides = {}) { + return { + storeNum: '782', + site: { id: 'site-1', name: 'CG00782', storeNum: '782', description: '' }, + elements: [{ id: 'el-1', name: 'ION-A', model: 'ION1000', connected: true }], + healthscore: { value: 90, breakdown: {} }, + links: [ + { + interfaceId: 'if-mpls', interfaceName: 'wan1', + elementId: 'el-1', elementName: 'ION-A', + transportType: 'MPLS', + up: true, latencyMs: 40, jitterMs: 5, lossPct: 0.1, mos: 4.4, + }, + ], + alarms: { last1h: { critical: 0, major: 0, minor: 0 }, samples: [] }, + errors: [], + fetchedAt: new Date().toISOString(), + ...overrides, + }; +} + +test('renderer: no site → empty string (caller no-ops)', () => { + const md = renderWanDiagnosticsMarkdown({ site: null, storeNum: '782' }); + assert.equal(md, ''); +}); + +test('renderer: null / non-object → empty string', () => { + assert.equal(renderWanDiagnosticsMarkdown(null), ''); + assert.equal(renderWanDiagnosticsMarkdown('nope'), ''); +}); + +test('renderer: healthy site — header + site + one path bullet', () => { + const md = renderWanDiagnosticsMarkdown(baseData(), { storeNum: '782' }); + assert.match(md, /WAN Diagnostics.*Store 782/); + assert.match(md, /Site.*CG00782/); + assert.match(md, /Healthscore.*90/); + assert.match(md, /wan1/); + assert.match(md, /MPLS/); + assert.match(md, /latency 40ms/); + assert.match(md, /MOS 4\.4/); +}); + +test('renderer: degraded path — worst path floats to top', () => { + const data = baseData({ + links: [ + { interfaceId: 'a', interfaceName: 'good', up: true, latencyMs: 30, jitterMs: 3, lossPct: 0, mos: 4.5 }, + { interfaceId: 'b', interfaceName: 'bad', up: true, latencyMs: 500, jitterMs: 60, lossPct: 5, mos: 3.0 }, + ], + }); + const md = renderWanDiagnosticsMarkdown(data, { storeNum: '782' }); + const goodIdx = md.indexOf('good'); + const badIdx = md.indexOf('bad'); + assert.ok(badIdx >= 0 && goodIdx >= 0); + assert.ok(badIdx < goodIdx, 'bad link should render before good link'); +}); + +test('renderer: all links down — down status labelled per link', () => { + const data = baseData({ + links: [ + { interfaceId: 'a', interfaceName: 'mpls', up: false, latencyMs: null, jitterMs: null, lossPct: null, mos: null }, + { interfaceId: 'b', interfaceName: 'broadband', up: false, latencyMs: null, jitterMs: null, lossPct: null, mos: null }, + ], + }); + const md = renderWanDiagnosticsMarkdown(data, { storeNum: '782' }); + assert.match(md, /mpls.*DOWN/); + assert.match(md, /broadband.*DOWN/); +}); + +test('renderer: partial-fetch errors surfaced under "Partial fetch"', () => { + const data = baseData({ + errors: [ + { scope: 'lqm.latency', message: 'timeout' }, + { scope: 'alarms', message: 'unauthorized' }, + ], + }); + const md = renderWanDiagnosticsMarkdown(data, { storeNum: '782' }); + assert.match(md, /Partial fetch/); + assert.match(md, /lqm\.latency.*timeout/); + assert.match(md, /alarms.*unauthorized/); +}); + +test('renderer: alarms summary emitted when non-zero — sample lines show code, not raw JSON', () => { + const data = baseData({ + alarms: { + last1h: { critical: 1, major: 2, minor: 3 }, + samples: [ + { code: 'NETWORK_ANYNETLINK_DOWN', message: 'link flapping', severity: 'critical', ts: new Date().toISOString() }, + ], + }, + }); + const md = renderWanDiagnosticsMarkdown(data, { storeNum: '782' }); + assert.match(md, /Alarms.*1 critical, 2 major, 3 minor/); + assert.match(md, /NETWORK_ANYNETLINK_DOWN/); + // Rendered rollup lines must NEVER include raw JSON blobs. If we + // regress and paste a stringified `info` dict into chat, this catches it. + assert.equal(md.includes('{"vpn_reasons'), false, 'no raw JSON in rendered alarm samples'); + assert.equal(md.includes('[object Object]'), false, 'no ugly [object Object] in rendered alarm samples'); +}); + +test('renderer: alarm rollup collapses N identical (code, severity) pairs to one line with ×N', () => { + // Regression: previously each of the 20 NETWORK_ANYNETLINK_DOWN + // events was pasted into chat as its own JSON blob line — noisy + // and unreadable. Rollup by (code, severity) prints one line. + const now = Date.now(); + const dupes = Array.from({ length: 20 }, (_, i) => ({ + code: 'NETWORK_ANYNETLINK_DOWN', + severity: 'major', + message: 'noise', + ts: new Date(now - i * 1000).toISOString(), + })); + const data = baseData({ + alarms: { + last1h: { critical: 0, major: 20, minor: 0 }, + samples: dupes, + }, + }); + const md = renderWanDiagnosticsMarkdown(data, { storeNum: '782' }); + // Exactly one line containing the code with the ×20 count marker. + const matches = md.match(/NETWORK_ANYNETLINK_DOWN/g) || []; + assert.equal(matches.length, 1, 'code should appear exactly once after rollup'); + assert.match(md, /×20/, 'should print a count marker for the rollup'); +}); + +test('renderer: alarm rollup orders by severity (critical → major → minor)', () => { + const data = baseData({ + alarms: { + last1h: { critical: 1, major: 1, minor: 1 }, + samples: [ + { code: 'MINOR_CODE', severity: 'minor', ts: 't1' }, + { code: 'MAJOR_CODE', severity: 'major', ts: 't2' }, + { code: 'CRITICAL_CODE', severity: 'critical', ts: 't3' }, + ], + }, + }); + const md = renderWanDiagnosticsMarkdown(data, { storeNum: '782' }); + const iCrit = md.indexOf('CRITICAL_CODE'); + const iMaj = md.indexOf('MAJOR_CODE'); + const iMinor = md.indexOf('MINOR_CODE'); + assert.ok(iCrit > 0 && iMaj > 0 && iMinor > 0); + assert.ok(iCrit < iMaj, 'critical printed before major'); + assert.ok(iMaj < iMinor, 'major printed before minor'); +}); + +test('renderer: no alarms — no alarm section rendered', () => { + const md = renderWanDiagnosticsMarkdown(baseData(), { storeNum: '782' }); + // The section header would be "Alarms (last 1h):"; the footer + // mentions wanAlarms as a --only check name which is fine. + assert.equal(md.includes('Alarms (last 1h)'), false); +}); + +test('renderer: no links — placeholder line', () => { + const md = renderWanDiagnosticsMarkdown(baseData({ links: [] }), { storeNum: '782' }); + assert.match(md, /No WAN path metrics available/); +}); + +test('renderer: healthscore missing → "n/a" with unknown icon', () => { + const md = renderWanDiagnosticsMarkdown(baseData({ healthscore: null }), { storeNum: '782' }); + assert.match(md, /Healthscore.*n\/a/); +}); diff --git a/tests/sdwanEnrichment.test.js b/tests/sdwanEnrichment.test.js new file mode 100644 index 0000000..065edc1 --- /dev/null +++ b/tests/sdwanEnrichment.test.js @@ -0,0 +1,844 @@ +// tests/sdwanEnrichment.test.js +// +// Split into two flavors: +// 1) Unit tests on the exported parse* / buildLinkRows helpers — +// these are pure functions, no HTTP. +// 2) Integration test of `collectSdwanForStore` end-to-end using a +// fake Prisma HTTP server. This exercises the site lookup + +// element fetch + parallel metric composition + `errors[]` +// preservation on partial failure. + +import test from 'node:test'; +import assert from 'node:assert/strict'; +import http from 'node:http'; + +import { + collectSdwanForStore, + parseHealthscore, + buildLinkRows, + parseAlarms, +} from '../services/enrichment/sdwanEnrichment.js'; +import { _resetSitesCache } from '../integrations/paloalto/sites.js'; +import { _resetPrismaAuthCache } from '../integrations/paloalto/client.js'; + +// ─── Unit tests: parseHealthscore ─────────────────────────────────── + +test('parseHealthscore: nil / malformed inputs → null', () => { + assert.equal(parseHealthscore(null), null); + assert.equal(parseHealthscore({}), null); + assert.equal(parseHealthscore({ metrics: [] }), null); + assert.equal(parseHealthscore({ metrics: 'nope' }), null); +}); + +test('parseHealthscore: LIVE v2.6 shape — metrics[].series[].data[].datapoints[].value (WINNER)', () => { + // The actual live shape verified 2026-07-09 on tenant. Note the + // wrapper: data[0] is {statistics: 'max', datapoints: [...]}, NOT + // the array of samples itself. + const resp = { + metrics: [{ + series: [{ + name: 'Healthscore', + unit: 'gauge', + interval: '5min', + view: 'summary', + data: [{ + statistics: 'max', + datapoints: [ + { time: '2026-07-09T12:45:00Z', value: 100 }, + { time: '2026-07-09T12:50:00Z', value: 98 }, + { time: '2026-07-09T12:55:00Z', value: 92 }, + { time: '2026-07-09T13:00:00Z', value: 87 }, + ], + }], + }], + }], + }; + const hs = parseHealthscore(resp, '16158173173100144'); + assert.equal(hs.value, 87, 'must pick the LAST datapoint in the wrapper'); + assert.deepEqual(hs.breakdown, {}); +}); + +test('parseHealthscore: LIVE v2.6 shape — empty datapoints → falls through to next shape', () => { + const resp = { + metrics: [{ + series: [{ + name: 'Healthscore', + data: [{ statistics: 'max', datapoints: [] }], + }], + }], + }; + assert.equal(parseHealthscore(resp, 'target-site'), null, + 'empty datapoints must not silently return a stale/undefined value'); +}); + +test('parseHealthscore: FALLBACK v2.6 shape — metrics[0].sites[].healthscore', () => { + const resp = { + metrics: [{ + name: 'Healthscore', + sites: [ + { site_id: 'other-1', healthscore: 55 }, + { site_id: 'target-site', healthscore: 92 }, + ], + }], + }; + const hs = parseHealthscore(resp, 'target-site'); + assert.equal(hs.value, 92, 'must pick the site entry whose site_id matches'); +}); + +test('parseHealthscore: LIVE v2.6 shape — nested under .data.score', () => { + const resp = { + metrics: [{ + name: 'Healthscore', + sites: [ + { site_id: 'target-site', data: { score: 87 } }, + ], + }], + }; + const hs = parseHealthscore(resp, 'target-site'); + assert.equal(hs.value, 87); +}); + +test('parseHealthscore: LIVE v2.6 shape — fallback picks any 0-100 numeric field', () => { + // Defensive: if Prisma renames the value field between versions, + // parseHealthscore falls through to the first numeric in 0-100 range. + const resp = { + metrics: [{ + name: 'Healthscore', + sites: [{ site_id: 'target-site', mystery_key: 73, unrelated_id: 999 }], + }], + }; + const hs = parseHealthscore(resp, 'target-site'); + assert.equal(hs.value, 73, 'unrelated_id (999) skipped because outside 0-100'); +}); + +test('parseHealthscore: LEGACY shape — metrics[0].series[].data[].value (pan.dev-documented)', () => { + // Fallback support for other tenants that DO return the pan.dev + // shape. This tenant doesn't, but we don't want to break others. + const resp = { + metrics: [{ + name: 'Healthscore', + series: [ + { view: { site: 'wrong-1' }, data: [{ value: 40 }] }, + { view: { site: 'target-site' }, data: [{ value: 91 }] }, + ], + }], + }; + const hs = parseHealthscore(resp, 'target-site'); + assert.equal(hs.value, 91); +}); + +test('parseHealthscore: no siteId → first available (backwards-compat)', () => { + const resp = { + metrics: [{ + name: 'Healthscore', + sites: [ + { site_id: 'first', healthscore: 77 }, + { site_id: 'second', healthscore: 33 }, + ], + }], + }; + const hs = parseHealthscore(resp); + assert.equal(hs.value, 77); +}); + +// ─── Unit tests: buildLinkRows ────────────────────────────────────── +// +// The waninterface config list is the source of truth for the row +// set. LQM metrics are layered in as overlays keyed by +// `view.waninterface`. `up` comes from the waninterface's admin +// state (the closest we have to a runtime up/down signal until we +// wire /waninterfaces/{id}/status per element in a later phase). + +/** + * Build a live-shape LQM response (metrics[].sites[].paths[].data.). + * This is the shape verified against a real tenant on 2026-07-09 via + * /scripts/prismaProbe.js — not the pan.dev-documented shape. + */ +function lqmLiveResp({ siteId = 'site-A', pathId, dataKey, value, completeness = 100 }) { + return { + metrics: [{ + name: 'LqmLatencyPointMetric', + unit: 'milliseconds', + sites: [{ + site_id: siteId, + paths: [{ + path_id: pathId, + remote_site_id: '0', + data: { + sample_completeness: completeness, + [dataKey]: value, + }, + }], + }], + }], + }; +} + +// Legacy shape (pan.dev-documented) — kept as a fallback test target +// so we notice if a future tenant returns this variant. +function lqmLegacySeries(waninterfaceId, points) { + return { view: { waninterface: waninterfaceId }, data: points }; +} + +test('buildLinkRows: LIVE shape — metrics[].sites[].paths[].data.', () => { + const rows = buildLinkRows({ + wanInterfaces: [ + { id: 'wi-mpls', name: 'MPLS Circuit', adminUp: true, usedFor: 'primary' }, + { id: 'wi-bb', name: 'Broadband', adminUp: true, usedFor: 'secondary' }, + ], + metricResponses: { + latency: lqmLiveResp({ pathId: 'wi-mpls', dataKey: 'rtt_latency', value: 42.3 }), + jitter: lqmLiveResp({ pathId: 'wi-mpls', dataKey: 'rtt_jitter', value: 8.7 }), + loss: lqmLiveResp({ pathId: 'wi-mpls', dataKey: 'pkt_loss_pct', value: 0.4 }), + mos: lqmLiveResp({ pathId: 'wi-mpls', dataKey: 'mos', value: 4.42 }), + }, + }); + assert.equal(rows.length, 2); + const mpls = rows.find((r) => r.interfaceId === 'wi-mpls'); + assert.equal(mpls.interfaceName, 'MPLS Circuit'); + assert.equal(mpls.up, true); + assert.equal(mpls.latencyMs, 42.3); + assert.equal(mpls.jitterMs, 8.7); + assert.equal(mpls.lossPct, 0.4); + assert.equal(mpls.mos, 4.42); + assert.equal(mpls.transportType, 'primary'); + assert.equal(mpls.sampleCompleteness, 100, 'quality marker preserved for diagnostics'); + + const bb = rows.find((r) => r.interfaceId === 'wi-bb'); + assert.equal(bb.up, true); + assert.equal(bb.latencyMs, null, 'broadband had no metric samples'); +}); + +test('buildLinkRows: LIVE shape — matches paths by path_id (not view.path)', () => { + // Regression: initial parser looked for series[].view.path, missing + // the fact that Prisma's live tenant returns a completely different + // metrics[].sites[].paths[].path_id shape. Verified live 2026-07-09. + const rows = buildLinkRows({ + wanInterfaces: [{ id: '16158173176610209', name: 'Inet1', adminUp: null, usedFor: null }], + metricResponses: { + latency: lqmLiveResp({ + siteId: '16158173173100144', + pathId: '16158173176610209', + dataKey: 'rtt_latency', + value: 22.0121008, + }), + }, + }); + assert.equal(rows.length, 1); + assert.equal(rows[0].latencyMs, 22, 'rounded to 1 decimal'); +}); + +test('buildLinkRows: LIVE shape — loss is directional (max of downlink + uplink)', () => { + // Prisma returns packet loss as two separate keys — one per + // direction — and the fallback scanner would pick whichever + // came first, silently under-reporting asymmetric loss patterns. + // We must fold them via max(). Verified live 2026-07-09 via + // `try-shapes lqm-loss` which returned keys: + // data.downlink_pkt_loss_avg and data.uplink_pkt_loss_avg + const rows = buildLinkRows({ + wanInterfaces: [{ id: 'wi-1', name: 'w1', adminUp: true, usedFor: 'primary' }], + metricResponses: { + loss: { + metrics: [{ + name: 'LqmPktLossPointMetric', + sites: [{ + site_id: 'site-A', + paths: [{ + path_id: 'wi-1', + data: { + sample_completeness: 100, + downlink_pkt_loss_avg: 0.0, + uplink_pkt_loss_avg: 2.5, + }, + }], + }], + }], + }, + }, + }); + assert.equal(rows.length, 1); + assert.equal(rows[0].lossPct, 2.5, + 'must report the WORSE direction (2.5%) — reporting 0 would hide upstream loss'); +}); + +test('buildLinkRows: LIVE shape — MOS is directional (MIN of downlink + uplink avg)', () => { + // Prisma returns MOS as 6 keys per path: + // downlink_mos_{avg,min,max} + uplink_mos_{avg,min,max} + // Lower MOS = worse audio, so if downlink is 4.5 (great) but + // uplink is 3.0 (barely usable), the call sounds bad and we must + // surface 3.0. Picking any single key (like the fallback did) + // would hide the problem. Verified live via `try-shapes lqm-mos` + // 2026-07-09. + const rows = buildLinkRows({ + wanInterfaces: [{ id: 'wi-1', name: 'w1', adminUp: true, usedFor: 'primary' }], + metricResponses: { + mos: { + metrics: [{ + name: 'LqmMosPointMetric', + sites: [{ + site_id: 'site-A', + paths: [{ + path_id: 'wi-1', + data: { + sample_completeness: 100, + downlink_mos_avg: 4.5, + downlink_mos_min: 4.2, + downlink_mos_max: 4.6, + uplink_mos_avg: 3.0, + uplink_mos_min: 2.7, + uplink_mos_max: 3.3, + }, + }], + }], + }], + }, + }, + }); + assert.equal(rows.length, 1); + assert.equal(rows[0].mos, 3.0, + 'must report the WORSE direction avg (3.0) — reporting downlink 4.5 would hide bad-uplink calls'); +}); + +test('buildLinkRows: LIVE shape — MOS with symmetric high values → picks the min', () => { + const rows = buildLinkRows({ + wanInterfaces: [{ id: 'wi-1', name: 'w1', adminUp: true, usedFor: 'primary' }], + metricResponses: { + mos: { + metrics: [{ + name: 'LqmMosPointMetric', + sites: [{ + site_id: 'site-A', + paths: [{ + path_id: 'wi-1', + data: { + sample_completeness: 100, + downlink_mos_avg: 4.39, + uplink_mos_avg: 4.42, + }, + }], + }], + }], + }, + }, + }); + assert.equal(rows[0].mos, 4.39, + 'both directions healthy → still take the worse (lower) direction'); +}); + +test('buildLinkRows: LIVE shape — loss with symmetric zero on both directions → 0 (not null)', () => { + const rows = buildLinkRows({ + wanInterfaces: [{ id: 'wi-1', name: 'w1', adminUp: true, usedFor: 'primary' }], + metricResponses: { + loss: { + metrics: [{ + name: 'LqmPktLossPointMetric', + sites: [{ + site_id: 'site-A', + paths: [{ + path_id: 'wi-1', + data: { + sample_completeness: 100, + downlink_pkt_loss_avg: 0, + uplink_pkt_loss_avg: 0, + }, + }], + }], + }], + }, + }, + }); + assert.equal(rows[0].lossPct, 0, 'zero loss must render as 0, NOT null'); +}); + +test('buildLinkRows: LIVE shape — extractLqmValue fallback on unknown data key', () => { + // Defensive: if Prisma renames a data key between tenant versions, + // extractLqmValue falls back to the first numeric non-completeness + // field. Simulate that by using an unknown key name. + const rows = buildLinkRows({ + wanInterfaces: [{ id: 'wi-1', name: 'w1', adminUp: true, usedFor: 'primary' }], + metricResponses: { + latency: { + metrics: [{ + name: 'LqmLatencyPointMetric', + sites: [{ + site_id: 'site-A', + paths: [{ path_id: 'wi-1', data: { sample_completeness: 100, mystery_new_key: 55 } }], + }], + }], + }, + }, + }); + assert.equal(rows.length, 1); + assert.equal(rows[0].latencyMs, 55, 'fallback picked the first numeric non-completeness field'); +}); + +test('buildLinkRows: LEGACY shape (pan.dev) — metrics[].series[].data[].value with view.path', () => { + // Fallback shape support — should keep working for any tenant that + // returns the pan.dev-documented response. + const rows = buildLinkRows({ + wanInterfaces: [{ id: 'wi-mpls', name: 'MPLS', adminUp: true, usedFor: 'primary' }], + metricResponses: { + latency: { metrics: [{ series: [{ view: { path: 'wi-mpls' }, data: [{ value: 25 }] }] }] }, + }, + }); + assert.equal(rows.length, 1); + assert.equal(rows[0].latencyMs, 25); +}); + +test('buildLinkRows: LEGACY shape — last non-null sample wins', () => { + const rows = buildLinkRows({ + wanInterfaces: [{ id: 'wi-1', name: 'w1', adminUp: true, usedFor: 'primary' }], + metricResponses: { + latency: { metrics: [{ series: [ + lqmLegacySeries('wi-1', [{ value: 10 }, { value: null }, { value: 20 }, { value: null }]), + ]}]}, + }, + }); + assert.equal(rows.length, 1); + assert.equal(rows[0].latencyMs, 20); +}); + +test('buildLinkRows: LQM samples for unknown path_id are dropped', () => { + const rows = buildLinkRows({ + wanInterfaces: [{ id: 'wi-1', name: 'w1', adminUp: true, usedFor: 'primary' }], + metricResponses: { + latency: lqmLiveResp({ pathId: 'wi-orphan', dataKey: 'rtt_latency', value: 99 }), + }, + }); + assert.equal(rows.length, 1); + assert.equal(rows[0].interfaceId, 'wi-1'); + assert.equal(rows[0].latencyMs, null, 'no sample matched the configured wan interface'); +}); + +test('buildLinkRows: no waninterfaces → empty list even with LQM data', () => { + const rows = buildLinkRows({ + wanInterfaces: [], + metricResponses: { + latency: lqmLiveResp({ pathId: 'wi-1', dataKey: 'rtt_latency', value: 10 }), + }, + }); + assert.deepEqual(rows, []); +}); + +test('buildLinkRows: admin-down waninterface reports up:false', () => { + const rows = buildLinkRows({ + wanInterfaces: [{ id: 'wi-lte', name: 'LTE', adminUp: false, usedFor: 'backup' }], + metricResponses: {}, + }); + assert.equal(rows.length, 1); + assert.equal(rows[0].up, false); +}); + +test('buildLinkRows: adminUp:null but WITH LQM samples → up:true (data-flowing == up)', () => { + // Regression: on the live tenant, admin_up isn't exposed by the + // waninterface config API — every row starts with up=null. But if + // Prisma returns LQM samples for a path, that path is provably + // carrying traffic. Since traffic == up, we upgrade the row. + // Otherwise the WAN Link State check flags every path as unknown + // even when circuits are visibly healthy. + const rows = buildLinkRows({ + wanInterfaces: [{ id: 'wi-x', name: 'Inet1', adminUp: null, usedFor: 'primary' }], + metricResponses: { + latency: lqmLiveResp({ pathId: 'wi-x', dataKey: 'rtt_latency', value: 22.3 }), + }, + }); + assert.equal(rows.length, 1); + assert.equal(rows[0].up, true, 'LQM samples prove the path is carrying traffic'); + assert.equal(rows[0].latencyMs, 22.3); +}); + +test('buildLinkRows: adminUp:null WITHOUT LQM samples → up:null (still unknown)', () => { + // Complement of the previous test: if we have NEITHER admin_up + // NOR LQM samples, we genuinely don't know the state and must + // NOT lie about it. + const rows = buildLinkRows({ + wanInterfaces: [{ id: 'wi-x', name: 'Inet1', adminUp: null, usedFor: 'primary' }], + metricResponses: {}, + }); + assert.equal(rows.length, 1); + assert.equal(rows[0].up, null, 'null (unknown) is meaningfully different from false (admin-down)'); +}); + +test('buildLinkRows: adminUp:false → up:false even when LQM samples arrive (admin-disabled wins)', () => { + // Safety: an admin-disabled circuit is DOWN regardless of what + // Prisma monitoring says. Never upgrade explicit adminUp=false. + const rows = buildLinkRows({ + wanInterfaces: [{ id: 'wi-lte', name: 'LTE', adminUp: false, usedFor: 'backup' }], + metricResponses: { + latency: lqmLiveResp({ pathId: 'wi-lte', dataKey: 'rtt_latency', value: 45 }), + }, + }); + assert.equal(rows.length, 1); + assert.equal(rows[0].up, false, 'admin-disabled must remain down even with monitor samples'); + assert.equal(rows[0].latencyMs, 45, 'samples still recorded for diagnostics'); +}); + +// ─── Unit tests: parseAlarms ──────────────────────────────────────── + +test('parseAlarms: nil → empty counts', () => { + const a = parseAlarms(null); + assert.deepEqual(a.last1h, { critical: 0, major: 0, minor: 0 }); + assert.deepEqual(a.samples, []); +}); + +test('parseAlarms: counts by severity, keeps 5 most-recent samples', () => { + const resp = { + items: [ + { severity: 'critical', code: 'C1', info: 'boom', time: '2025-01-01T10:00:00Z' }, + { severity: 'major', code: 'M1', info: 'wobble', time: '2025-01-01T09:00:00Z' }, + { severity: 'major', code: 'M2', info: 'wobble2', time: '2025-01-01T08:00:00Z' }, + { severity: 'minor', code: 'm1', info: 'meh', time: '2025-01-01T07:00:00Z' }, + { severity: 'minor', code: 'm2', info: 'meh2', time: '2025-01-01T06:00:00Z' }, + { severity: 'minor', code: 'm3', info: 'meh3', time: '2025-01-01T05:00:00Z' }, + ], + }; + const a = parseAlarms(resp); + assert.equal(a.last1h.critical, 1); + assert.equal(a.last1h.major, 2); + assert.equal(a.last1h.minor, 3); + assert.equal(a.samples.length, 5); + assert.equal(a.samples[0].code, 'C1', 'newest first'); +}); + +test('parseAlarms: filters out cleared alarms (events/query returns both open + cleared)', () => { + const resp = { + items: [ + { severity: 'critical', code: 'STILL_OPEN', info: 'x', time: 't1' }, + { severity: 'critical', code: 'ALREADY_FIXED', info: 'y', time: 't2', cleared: true }, + { severity: 'major', code: 'ALSO_CLEARED', info: 'z', time: 't3', cleared: true }, + ], + }; + const a = parseAlarms(resp); + assert.equal(a.last1h.critical, 1, 'only the still-open critical is counted'); + assert.equal(a.last1h.major, 0); + assert.equal(a.samples.length, 1); + assert.equal(a.samples[0].code, 'STILL_OPEN'); +}); + +test('parseAlarms: local site filter drops events whose site_id does not match', () => { + // Regression: Prisma's events/query sometimes silently ignores our + // `query.site` filter, in which case we get tenant-wide events. If + // we counted those as this-site alarms we'd report wildly inflated + // counts (e.g. 20 major for a small store). Local site match here + // is the safety net. + const resp = { + items: [ + { severity: 'critical', code: 'OUR_ALARM', site_id: 'target-site', time: 't1' }, + { severity: 'major', code: 'OTHER_STORE', site_id: 'other-site-1', time: 't2' }, + { severity: 'major', code: 'ALSO_OTHER', site_id: 'other-site-2', time: 't3' }, + // No site_id — tenant-scoped event, kept + { severity: 'minor', code: 'TENANT_WIDE', time: 't4' }, + ], + }; + const a = parseAlarms(resp, 'target-site'); + assert.equal(a.last1h.critical, 1); + assert.equal(a.last1h.major, 0, 'other-store alarms must NOT count toward this site'); + assert.equal(a.last1h.minor, 1, 'events with no site_id are kept (tenant-scoped)'); + assert.equal(a._diag.rawEventCount, 4); + assert.equal(a._diag.droppedForSiteMismatch, 2); +}); + +test('parseAlarms: no siteId argument → no local scoping (backwards-compat)', () => { + const resp = { + items: [ + { severity: 'critical', code: 'A', site_id: 'anything', time: 't1' }, + { severity: 'major', code: 'B', site_id: 'something-else', time: 't2' }, + ], + }; + const a = parseAlarms(resp); + assert.equal(a.last1h.critical, 1); + assert.equal(a.last1h.major, 1, 'no siteId → do not drop by site'); + assert.equal(a._diag.droppedForSiteMismatch, 0); +}); + +test('parseAlarms: nested info object → JSON-stringified (not "[object Object]")', () => { + // Live Prisma events regularly carry `info` as a nested object, + // e.g. { vpn_link_id: '...' }. If we let String() coerce this we + // dump [object Object] into chat. Flatten to JSON instead. + const resp = { + items: [ + { severity: 'major', code: 'ANYNET_DOWN', info: { vpn_link_id: 'link-1' }, time: 't' }, + ], + }; + const a = parseAlarms(resp); + assert.equal(a.samples[0].message.startsWith('{'), true, + 'nested object should be JSON-stringified for renderer safety'); + assert.match(a.samples[0].message, /vpn_link_id/); +}); + +// ─── Integration: collectSdwanForStore end-to-end ─────────────────── + +async function makeFakePrisma(handlers) { + const server = http.createServer((req, res) => { + let body = ''; + req.on('data', (c) => (body += c)); + req.on('end', () => { + if (req.url === '/oauth2/access_token') { + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ access_token: 't', expires_in: 900 })); + return; + } + // Mandatory SASE unified SD-WAN session priming call. + if (req.url === '/sdwan/v2.1/api/profile' && req.method === 'GET') { + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ id: 'stub-profile' })); + return; + } + // Path-only fallback so tests don't have to encode ?limit=1000 + // etc. in their handler keys. + const pathOnly = (req.url || '').split('?')[0]; + const h = handlers[`${req.method} ${req.url}`] || handlers[`${req.method} ${pathOnly}`]; + if (h) { + const parsed = body ? JSON.parse(body) : null; + const r = h({ req, body: parsed }); + res.writeHead(r.status || 200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify(r.body || {})); + return; + } + res.writeHead(404); + res.end(); + }); + }); + await new Promise((r) => server.listen(0, '127.0.0.1', r)); + const { port } = server.address(); + return { baseUrl: `http://127.0.0.1:${port}`, close: () => new Promise((r) => server.close(r)) }; +} + +function setupSaseEnv(baseUrl) { + process.env.PRISMA_AUTH_MODE = 'sase'; + process.env.PRISMA_SASE_BASE_URL = baseUrl; + process.env.PRISMA_AUTH_URL = `${baseUrl}/oauth2/access_token`; + process.env.PRISMA_CLIENT_ID = 'id'; + process.env.PRISMA_CLIENT_SECRET = 'secret'; + process.env.PRISMA_TSG_ID = 'tsg'; +} +function clearEnv() { + ['PRISMA_AUTH_MODE','PRISMA_SASE_BASE_URL','PRISMA_AUTH_URL', + 'PRISMA_CLIENT_ID','PRISMA_CLIENT_SECRET','PRISMA_TSG_ID', + 'PRISMA_LEGACY_BASE_URL','PRISMA_EMAIL','PRISMA_PASSWORD'] + .forEach((k) => delete process.env[k]); +} + +test('collectSdwanForStore: no site → short-circuits with site:null', async () => { + _resetSitesCache(); _resetPrismaAuthCache(); + const fake = await makeFakePrisma({ + 'GET /sdwan/v4.13/api/sites': () => ({ + body: { items: [{ id: 's', name: 'CG99999' }] }, + }), + }); + setupSaseEnv(fake.baseUrl); + try { + const data = await collectSdwanForStore(782); + assert.equal(data.site, null); + assert.deepEqual(data.elements, []); + assert.deepEqual(data.links, []); + assert.equal(data.healthscore, null); + assert.equal(data.errors.length, 0); + assert.equal(data.storeNum, '782'); + } finally { + await fake.close(); _resetSitesCache(); _resetPrismaAuthCache(); clearEnv(); + } +}); + +test('collectSdwanForStore: happy path composes site+elements+waninterfaces+LQM', async () => { + _resetSitesCache(); _resetPrismaAuthCache(); + const fake = await makeFakePrisma({ + 'GET /sdwan/v4.13/api/sites': () => ({ + body: { items: [{ id: 'site-A', name: 'CG00782', description: 'store 782' }] }, + }), + 'GET /sdwan/v3.1/api/elements': () => ({ + body: { items: [{ id: 'el-1', name: 'ION-A', connected: true, site_id: 'site-A' }] }, + }), + 'GET /sdwan/v2.10/api/sites/site-A/waninterfaces': () => ({ + body: { items: [ + { id: 'wi-mpls', name: 'MPLS Circuit', admin_up: true, used_for: 'primary' }, + { id: 'wi-bb', name: 'Broadband', admin_up: true, used_for: 'secondary' }, + ]}, + }), + 'POST /sdwan/monitor/v2.6/api/monitor/metrics': () => ({ + // LIVE v2.6 healthscore response shape verified 2026-07-09. + // The response is already server-side scoped to filter.site, + // so no siteId filtering needed by parseHealthscore. + body: { + metrics: [{ + series: [{ + name: 'Healthscore', + unit: 'gauge', + interval: '5min', + view: 'summary', + data: [{ + statistics: 'max', + datapoints: [ + { time: '2026-07-09T12:45:00Z', value: 88 }, + { time: '2026-07-09T13:00:00Z', value: 92 }, + ], + }], + }], + }], + }, + }), + 'POST /sdwan/monitor/v2.0/api/monitor/lqm_point_metrics': ({ body }) => { + const name = body?.metrics?.[0]?.name; + // Live-tenant data shapes per metric (verified 2026-07-09): + // latency/jitter/mos → single scalar under RTT/aggregate key + // loss → TWO keys, directional. buildLinkRows + // folds via max(downlink, uplink). + const dataFor = { + LqmLatencyPointMetric: { rtt_latency: 30 }, + LqmJitterPointMetric: { rtt_jitter: 5 }, + LqmPktLossPointMetric: { downlink_pkt_loss_avg: 0.05, uplink_pkt_loss_avg: 0.1 }, + // MOS is directional × 3 stats. buildLinkRows takes + // min(downlink_mos_avg, uplink_mos_avg) — the worst average + // direction — so with downlink=4.35 + uplink=4.3 the row + // should end up at 4.3. + LqmMosPointMetric: { + downlink_mos_avg: 4.35, + downlink_mos_min: 4.2, + downlink_mos_max: 4.4, + uplink_mos_avg: 4.3, + uplink_mos_min: 4.1, + uplink_mos_max: 4.4, + }, + }; + return { + body: { + metrics: [{ + name, + unit: body?.metrics?.[0]?.unit, + sites: [{ + site_id: 'site-A', + paths: [{ + path_id: 'wi-mpls', + remote_site_id: '0', + data: { sample_completeness: 100, ...(dataFor[name] || {}) }, + }], + }], + }], + }, + }; + }, + 'POST /sdwan/v3.7/api/events/query': () => ({ + body: { items: [] }, + }), + }); + setupSaseEnv(fake.baseUrl); + try { + const data = await collectSdwanForStore(782); + assert.equal(data.site?.name, 'CG00782'); + assert.equal(data.elements.length, 1); + assert.equal(data.healthscore?.value, 92); + assert.equal(data.links.length, 2, 'one row per waninterface'); + const mpls = data.links.find((l) => l.interfaceId === 'wi-mpls'); + assert.equal(mpls.up, true); + assert.equal(mpls.latencyMs, 30); + assert.equal(mpls.jitterMs, 5); + assert.equal(mpls.lossPct, 0.1); + assert.equal(mpls.mos, 4.3); + assert.equal(mpls.transportType, 'primary'); + const bb = data.links.find((l) => l.interfaceId === 'wi-bb'); + assert.equal(bb.up, true); + assert.equal(bb.latencyMs, null, 'broadband has no LQM sample in the fixture'); + assert.equal(data.errors.length, 0); + } finally { + await fake.close(); _resetSitesCache(); _resetPrismaAuthCache(); clearEnv(); + } +}); + +test('collectSdwanForStore: reports the effective window on the returned payload', async () => { + _resetSitesCache(); _resetPrismaAuthCache(); + const fake = await makeFakePrisma({ + 'GET /sdwan/v4.13/api/sites': () => ({ + body: { items: [{ id: 's', name: 'CG00782' }] }, + }), + 'GET /sdwan/v3.1/api/elements': () => ({ + body: { items: [] }, + }), + 'GET /sdwan/v2.10/api/sites/s/waninterfaces': () => ({ + body: { items: [] }, + }), + 'POST /sdwan/monitor/v2.6/api/monitor/metrics': () => ({ + body: { metrics: [] }, + }), + 'POST /sdwan/monitor/v2.0/api/monitor/lqm_point_metrics': () => ({ + body: { metrics: [] }, + }), + 'POST /sdwan/v3.7/api/events/query': () => ({ + body: { items: [] }, + }), + }); + setupSaseEnv(fake.baseUrl); + try { + // Default (no opts): 15 min (also the env default) + let data = await collectSdwanForStore(782); + assert.equal(data.window?.minutes, 15); + assert.equal(data.window?.alarmMinutes, 60, + 'alarms floor at 60m even when the requested window is smaller'); + + // Explicit 24h override → both windows go to 1440 + data = await collectSdwanForStore(782, { windowMinutes: 1440 }); + assert.equal(data.window?.minutes, 1440); + assert.equal(data.window?.alarmMinutes, 1440); + + // Out-of-range values are clamped to [1, 1440] + data = await collectSdwanForStore(782, { windowMinutes: 99999 }); + assert.equal(data.window?.minutes, 1440, 'capped at 24h'); + + data = await collectSdwanForStore(782, { windowMinutes: -5 }); + assert.equal(data.window?.minutes, 15, 'invalid → falls to env default'); + } finally { + await fake.close(); _resetSitesCache(); _resetPrismaAuthCache(); clearEnv(); + } +}); + +test('collectSdwanForStore: reports window even when site is unresolved', async () => { + _resetSitesCache(); _resetPrismaAuthCache(); + const fake = await makeFakePrisma({ + 'GET /sdwan/v4.13/api/sites': () => ({ + body: { items: [{ id: 's', name: 'CG99999' }] }, + }), + }); + setupSaseEnv(fake.baseUrl); + try { + const data = await collectSdwanForStore(782, { windowMinutes: 60 }); + assert.equal(data.site, null); + assert.equal(data.window?.minutes, 60); + } finally { + await fake.close(); _resetSitesCache(); _resetPrismaAuthCache(); clearEnv(); + } +}); + +test('collectSdwanForStore: preserves per-metric failure in errors[]', async () => { + _resetSitesCache(); _resetPrismaAuthCache(); + const fake = await makeFakePrisma({ + 'GET /sdwan/v4.13/api/sites': () => ({ + body: { items: [{ id: 'site-B', name: 'CG00782' }] }, + }), + 'GET /sdwan/v3.1/api/elements': () => ({ + body: { items: [{ id: 'el-1', name: 'ION', connected: true, site_id: 'site-B' }] }, + }), + 'GET /sdwan/v2.10/api/sites/site-B/waninterfaces': () => ({ + body: { items: [] }, + }), + 'POST /sdwan/monitor/v2.6/api/monitor/metrics': () => ({ + status: 500, body: { error: 'boom' }, + }), + 'POST /sdwan/monitor/v2.0/api/monitor/lqm_point_metrics': () => ({ + body: { metrics: [] }, + }), + 'POST /sdwan/v3.7/api/events/query': () => ({ + body: { items: [] }, + }), + }); + setupSaseEnv(fake.baseUrl); + try { + const data = await collectSdwanForStore(782); + // Site + elements arrived; healthscore null because upstream + // returned 500 (the metric wrapper catches + returns null with + // a log line — no errors[] entry, healthscore just stays null). + assert.equal(data.site?.name, 'CG00782'); + assert.equal(data.healthscore, null); + assert.equal(data.links.length, 0, 'no waninterfaces → no rows'); + } finally { + await fake.close(); _resetSitesCache(); _resetPrismaAuthCache(); clearEnv(); + } +}); diff --git a/tests/voiceDiag.checks.test.js b/tests/voiceDiag.checks.test.js index 42e048d..67e77d4 100644 --- a/tests/voiceDiag.checks.test.js +++ b/tests/voiceDiag.checks.test.js @@ -515,6 +515,38 @@ test('normalizeArg: nullish input passed through', async () => { assert.equal(normalizeArg(undefined), undefined); }); +test('parseWindowMinutes: unit-suffixed values (m/h/d) resolve to minutes', async () => { + const { parseWindowMinutes } = await import('../commands/voiceDiag.js'); + assert.equal(parseWindowMinutes('15m'), 15); + assert.equal(parseWindowMinutes('15min'), 15); + assert.equal(parseWindowMinutes('1h'), 60); + assert.equal(parseWindowMinutes('6h'), 360); + assert.equal(parseWindowMinutes('24h'), 1440); + assert.equal(parseWindowMinutes('1d'), 1440); + assert.equal(parseWindowMinutes('1day'), 1440); + assert.equal(parseWindowMinutes('2 hours'), 120); +}); + +test('parseWindowMinutes: bare integer treated as minutes', async () => { + const { parseWindowMinutes } = await import('../commands/voiceDiag.js'); + assert.equal(parseWindowMinutes('30'), 30); + assert.equal(parseWindowMinutes('1440'), 1440); +}); + +test('parseWindowMinutes: empty / null → undefined (falls to env default)', async () => { + const { parseWindowMinutes } = await import('../commands/voiceDiag.js'); + assert.equal(parseWindowMinutes(null), undefined); + assert.equal(parseWindowMinutes(undefined), undefined); + assert.equal(parseWindowMinutes(''), undefined); +}); + +test('parseWindowMinutes: unrecognised token → null (caller shows friendly error)', async () => { + const { parseWindowMinutes } = await import('../commands/voiceDiag.js'); + assert.equal(parseWindowMinutes('bogus'), null); + assert.equal(parseWindowMinutes('15x'), null); + assert.equal(parseWindowMinutes('h1'), null); +}); + test('buildRemediationRegistry: contains every declared remediation exactly once', () => { const registry = buildRemediationRegistry(); const declared = new Map(); diff --git a/tests/voiceDiag.wan.test.js b/tests/voiceDiag.wan.test.js new file mode 100644 index 0000000..5429e25 --- /dev/null +++ b/tests/voiceDiag.wan.test.js @@ -0,0 +1,348 @@ +// tests/voiceDiag.wan.test.js +// +// Unit coverage for the 8 WAN checks under services/voiceDiag/checks/wan/. +// Same pattern as tests/voiceDiag.checks.test.js — build a stub +// ctx with a `sdwanSite` + `sdwanData` (no HTTP, no Prisma) and +// assert each check's verdict. +// +// Every check gets: +// - happy path (data present, in the compliant range) +// - threshold breach (warn and/or error where applicable) +// - missing data (skipped with a clear reason) +// - kill-switch skip (WAN_STANDARD_ENABLED=false) + +import test from 'node:test'; +import assert from 'node:assert/strict'; + +import { wanSiteCheck } from '../services/voiceDiag/checks/wan/wanSite.js'; +import { wanHealthscoreCheck } from '../services/voiceDiag/checks/wan/wanHealthscore.js'; +import { wanLinkStateCheck } from '../services/voiceDiag/checks/wan/wanLinkState.js'; +import { wanLatencyCheck } from '../services/voiceDiag/checks/wan/wanLatency.js'; +import { wanJitterCheck } from '../services/voiceDiag/checks/wan/wanJitter.js'; +import { wanLossCheck } from '../services/voiceDiag/checks/wan/wanLoss.js'; +import { wanMosCheck } from '../services/voiceDiag/checks/wan/wanMos.js'; +import { wanAlarmsCheck } from '../services/voiceDiag/checks/wan/wanAlarms.js'; +import { CHECKS } from '../services/voiceDiag/checks/index.js'; + +// ─── Helpers ──────────────────────────────────────────────────────── + +function mkWanCtx({ site = { id: 'site-1', name: 'CG00782' }, links = [], healthscore = null, alarms = null, elements = [] } = {}) { + return { + storeNum: '782', + personId: null, + personLabel: 'store 782', + telephonyProfile: {}, + phoneStatus: null, + sdwanSite: site, + sdwanData: site ? { + storeNum: '782', + site, + elements, + links, + healthscore, + alarms: alarms || { last1h: { critical: 0, major: 0, minor: 0 }, samples: [] }, + errors: [], + } : null, + }; +} + +function link(overrides = {}) { + return { + interfaceId: 'if-mpls', + interfaceName: 'wan1', + elementId: 'el-1', + elementName: 'ION-A', + transportType: 'MPLS', + up: true, + latencyMs: 40, + jitterMs: 5, + lossPct: 0.1, + mos: 4.4, + ...overrides, + }; +} + +async function withKillSwitchOn(fn) { + const prev = process.env.WAN_STANDARD_ENABLED; + process.env.WAN_STANDARD_ENABLED = 'false'; + try { await fn(); } + finally { + if (prev === undefined) delete process.env.WAN_STANDARD_ENABLED; + else process.env.WAN_STANDARD_ENABLED = prev; + } +} + +// ─── wanSite ──────────────────────────────────────────────────────── + +test('wanSite: happy path → ok with site + element + link counts in message', async () => { + const ctx = mkWanCtx({ + elements: [{ id: 'el-1', connected: true }, { id: 'el-2', connected: false }], + links: [link(), link({ interfaceId: 'if-bb' })], + }); + const r = await wanSiteCheck.run(ctx); + assert.equal(r.status, 'ok'); + assert.match(r.message, /CG00782/); + assert.equal(r.details.elementCount, 2); + assert.equal(r.details.connectedElementCount, 1); + assert.equal(r.details.linkCount, 2); +}); + +test('wanSite: kill-switch skips', async () => { + await withKillSwitchOn(async () => { + const r = await wanSiteCheck.run(mkWanCtx({ elements: [{ id: 'x', connected: true }] })); + assert.equal(r.status, 'skipped'); + assert.match(r.message, /WAN_STANDARD_ENABLED/); + }); +}); + +// ─── wanHealthscore ───────────────────────────────────────────────── + +test('wanHealthscore: 92/100 → ok', async () => { + const r = await wanHealthscoreCheck.run(mkWanCtx({ healthscore: { value: 92, breakdown: {} } })); + assert.equal(r.status, 'ok'); +}); + +test('wanHealthscore: 70/100 → warn (< default warn 80)', async () => { + const r = await wanHealthscoreCheck.run(mkWanCtx({ healthscore: { value: 70, breakdown: {} } })); + assert.equal(r.status, 'warn'); + assert.equal(r.details.value, 70); +}); + +test('wanHealthscore: 40/100 → error (< default error 60)', async () => { + const r = await wanHealthscoreCheck.run(mkWanCtx({ healthscore: { value: 40, breakdown: {} } })); + assert.equal(r.status, 'error'); +}); + +test('wanHealthscore: env override adjusts thresholds', async () => { + const prev = process.env.WAN_STANDARD_HEALTHSCORE_WARN; + process.env.WAN_STANDARD_HEALTHSCORE_WARN = '95'; + try { + const r = await wanHealthscoreCheck.run(mkWanCtx({ healthscore: { value: 92 } })); + assert.equal(r.status, 'warn', '92 should warn once threshold is raised to 95'); + } finally { + if (prev === undefined) delete process.env.WAN_STANDARD_HEALTHSCORE_WARN; + else process.env.WAN_STANDARD_HEALTHSCORE_WARN = prev; + } +}); + +test('wanHealthscore: missing → skipped', async () => { + const r = await wanHealthscoreCheck.run(mkWanCtx({ healthscore: null })); + assert.equal(r.status, 'skipped'); +}); + +test('wanHealthscore: kill-switch skips', async () => { + await withKillSwitchOn(async () => { + const r = await wanHealthscoreCheck.run(mkWanCtx({ healthscore: { value: 40 } })); + assert.equal(r.status, 'skipped'); + assert.match(r.message, /WAN_STANDARD_ENABLED/); + }); +}); + +// ─── wanLinkState ─────────────────────────────────────────────────── + +test('wanLinkState: all up → ok', async () => { + const r = await wanLinkStateCheck.run(mkWanCtx({ + links: [link({ interfaceId: 'a', up: true }), link({ interfaceId: 'b', up: true })], + })); + assert.equal(r.status, 'ok'); + assert.equal(r.details.up, 2); +}); + +test('wanLinkState: one down → error, offender named', async () => { + const r = await wanLinkStateCheck.run(mkWanCtx({ + links: [ + link({ interfaceId: 'a', up: true }), + link({ interfaceId: 'b', up: false, interfaceName: 'bb1', transportType: 'BROADBAND' }), + ], + })); + assert.equal(r.status, 'error'); + assert.equal(r.details.down, 1); + assert.match(r.message, /bb1/); +}); + +test('wanLinkState: all unknown, none up → warn', async () => { + const r = await wanLinkStateCheck.run(mkWanCtx({ + links: [link({ up: null }), link({ interfaceId: 'x', up: null })], + })); + assert.equal(r.status, 'warn'); + assert.equal(r.details.unknown, 2); +}); + +test('wanLinkState: no links → skipped', async () => { + const r = await wanLinkStateCheck.run(mkWanCtx({ links: [] })); + assert.equal(r.status, 'skipped'); +}); + +// ─── wanLatency ───────────────────────────────────────────────────── + +test('wanLatency: all under 150ms → ok', async () => { + const r = await wanLatencyCheck.run(mkWanCtx({ links: [link({ latencyMs: 40 }), link({ latencyMs: 80 })] })); + assert.equal(r.status, 'ok'); +}); + +test('wanLatency: one path 220ms → warn (> 150, < 400)', async () => { + const r = await wanLatencyCheck.run(mkWanCtx({ + links: [link({ latencyMs: 40 }), link({ interfaceId: 'x', interfaceName: 'x1', latencyMs: 220 })], + })); + assert.equal(r.status, 'warn'); + assert.match(r.message, /x1/); + assert.match(r.message, /220ms/); +}); + +test('wanLatency: one path 500ms → error (> 400)', async () => { + const r = await wanLatencyCheck.run(mkWanCtx({ + links: [link({ latencyMs: 40 }), link({ interfaceId: 'x', interfaceName: 'x1', latencyMs: 500 })], + })); + assert.equal(r.status, 'error'); +}); + +test('wanLatency: env override adjusts thresholds', async () => { + const prev = process.env.WAN_STANDARD_LATENCY_WARN_MS; + process.env.WAN_STANDARD_LATENCY_WARN_MS = '50'; + try { + const r = await wanLatencyCheck.run(mkWanCtx({ links: [link({ latencyMs: 80 })] })); + assert.equal(r.status, 'warn', '80ms should warn when threshold is 50'); + } finally { + if (prev === undefined) delete process.env.WAN_STANDARD_LATENCY_WARN_MS; + else process.env.WAN_STANDARD_LATENCY_WARN_MS = prev; + } +}); + +test('wanLatency: no data → skipped', async () => { + const r = await wanLatencyCheck.run(mkWanCtx({ links: [link({ latencyMs: null })] })); + assert.equal(r.status, 'skipped'); +}); + +test('wanLatency: kill-switch skips', async () => { + await withKillSwitchOn(async () => { + const r = await wanLatencyCheck.run(mkWanCtx({ links: [link({ latencyMs: 500 })] })); + assert.equal(r.status, 'skipped'); + }); +}); + +// ─── wanJitter ────────────────────────────────────────────────────── + +test('wanJitter: 5ms → ok', async () => { + const r = await wanJitterCheck.run(mkWanCtx({ links: [link({ jitterMs: 5 })] })); + assert.equal(r.status, 'ok'); +}); + +test('wanJitter: 40ms → warn (> 30, < 50)', async () => { + const r = await wanJitterCheck.run(mkWanCtx({ links: [link({ jitterMs: 40 })] })); + assert.equal(r.status, 'warn'); +}); + +test('wanJitter: 100ms → error (> 50)', async () => { + const r = await wanJitterCheck.run(mkWanCtx({ links: [link({ jitterMs: 100 })] })); + assert.equal(r.status, 'error'); +}); + +// ─── wanLoss ──────────────────────────────────────────────────────── + +test('wanLoss: 0.1% → ok', async () => { + const r = await wanLossCheck.run(mkWanCtx({ links: [link({ lossPct: 0.1 })] })); + assert.equal(r.status, 'ok'); +}); + +test('wanLoss: 2% → warn (> 1, < 3)', async () => { + const r = await wanLossCheck.run(mkWanCtx({ links: [link({ lossPct: 2 })] })); + assert.equal(r.status, 'warn'); +}); + +test('wanLoss: 5% → error (> 3)', async () => { + const r = await wanLossCheck.run(mkWanCtx({ links: [link({ lossPct: 5 })] })); + assert.equal(r.status, 'error'); +}); + +// ─── wanMos ───────────────────────────────────────────────────────── +// MOS is inverted — low is bad. + +test('wanMos: 4.4 → ok (>= 4.0)', async () => { + const r = await wanMosCheck.run(mkWanCtx({ links: [link({ mos: 4.4 })] })); + assert.equal(r.status, 'ok'); +}); + +test('wanMos: 3.8 → warn (< 4.0, >= 3.5)', async () => { + const r = await wanMosCheck.run(mkWanCtx({ links: [link({ mos: 3.8 })] })); + assert.equal(r.status, 'warn'); +}); + +test('wanMos: 3.0 → error (< 3.5)', async () => { + const r = await wanMosCheck.run(mkWanCtx({ links: [link({ mos: 3.0 })] })); + assert.equal(r.status, 'error'); +}); + +// ─── wanAlarms ────────────────────────────────────────────────────── + +test('wanAlarms: none → ok', async () => { + const r = await wanAlarmsCheck.run(mkWanCtx()); + assert.equal(r.status, 'ok'); +}); + +test('wanAlarms: minor only → ok (informational)', async () => { + const r = await wanAlarmsCheck.run(mkWanCtx({ + alarms: { last1h: { critical: 0, major: 0, minor: 3 }, samples: [] }, + })); + assert.equal(r.status, 'ok'); + assert.equal(r.details.minor, 3); +}); + +test('wanAlarms: major → warn', async () => { + const r = await wanAlarmsCheck.run(mkWanCtx({ + alarms: { last1h: { critical: 0, major: 1, minor: 0 }, samples: [{ code: 'M1', message: 'wobble' }] }, + })); + assert.equal(r.status, 'warn'); + assert.match(r.message, /wobble/); +}); + +test('wanAlarms: critical → error', async () => { + const r = await wanAlarmsCheck.run(mkWanCtx({ + alarms: { last1h: { critical: 2, major: 0, minor: 0 }, samples: [{ code: 'C1', message: 'boom' }] }, + })); + assert.equal(r.status, 'error'); + assert.match(r.message, /boom/); +}); + +test('wanAlarms: no alarms feed → skipped', async () => { + const ctx = mkWanCtx(); + ctx.sdwanData.alarms = null; + const r = await wanAlarmsCheck.run(ctx); + assert.equal(r.status, 'skipped'); +}); + +// ─── standards regression + registry ordering ────────────────────── + +test('every WAN check exposes a standards object', () => { + const wanChecks = CHECKS.filter((c) => c.id.startsWith('wan')); + assert.equal(wanChecks.length, 8, 'expected 8 registered WAN checks'); + const missing = wanChecks.filter((c) => !c.standards || typeof c.standards !== 'object'); + assert.deepEqual(missing.map((c) => c.id), []); +}); + +test('WAN checks registered in the expected order after port bucket', () => { + const ids = CHECKS.map((c) => c.id); + const wanIds = ids.filter((id) => id.startsWith('wan')); + assert.deepEqual(wanIds, [ + 'wanSite', + 'wanHealthscore', + 'wanLinkState', + 'wanLatency', + 'wanJitter', + 'wanLoss', + 'wanMos', + 'wanAlarms', + ]); + // WAN bucket sits after the port bucket and before phoneOnline. + const portEnabledIdx = ids.indexOf('portEnabled'); + const wanSiteIdx = ids.indexOf('wanSite'); + const phoneOnlineIdx = ids.indexOf('phoneOnline'); + assert.ok(portEnabledIdx < wanSiteIdx); + assert.ok(wanSiteIdx < phoneOnlineIdx); +}); + +test('no WAN check declares a remediation (diagnostic-only)', () => { + const wanChecks = CHECKS.filter((c) => c.id.startsWith('wan')); + for (const c of wanChecks) { + assert.equal(c.remediations, undefined, `${c.id} should not expose remediations`); + } +}); diff --git a/tests/voiceDiagRenderer.test.js b/tests/voiceDiagRenderer.test.js index 0fdcc70..33eb461 100644 --- a/tests/voiceDiagRenderer.test.js +++ b/tests/voiceDiagRenderer.test.js @@ -158,3 +158,143 @@ test('renderer: skipped bucket shown even without detail', () => { assert.match(md, /\*\*SKIPPED\*\*/); assert.match(md, /403 missing scope/); }); + +// ─── Shape-aware detail formatters ────────────────────────────────── + +test('renderer: perLink shape → clean per-link list with verdict icons + threshold + roll-up', () => { + const results = [ + R('wanLatency', 'ok', 'All 3 paths ok', null, { + total: 3, ok: 3, warn: 0, error: 0, + warnThresh: 150, errorThresh: 400, + standardLabel: 'warn > 150ms, error > 400ms', + perLink: [ + { link: 'Inet1-00782', value: 22.2, interfaceId: '1', verdict: 'ok' }, + { link: 'Inet2-00782', value: 13.5, interfaceId: '2', verdict: 'ok' }, + { link: '5G-LTE-00782', value: 52.4, interfaceId: '3', verdict: 'ok' }, + ], + }), + ]; + const md = renderVoiceDiagMarkdown(results, { storeNum: '782', detailed: true }); + // Clean per-link list — one line per link with verdict icon + assert.match(md, /Threshold: warn > 150ms, error > 400ms/); + assert.match(md, /Per link:/); + assert.match(md, /✅ Inet1-00782: 22\.2/); + assert.match(md, /✅ Inet2-00782: 13\.5/); + assert.match(md, /✅ 5G-LTE-00782: 52\.4/); + assert.match(md, /Roll-up: 3 total · 3 ok · 0 warn · 0 error/); + // Must NOT dump the raw perLink JSON in a stringified form. + assert.equal(md.includes('"link":'), false); + assert.equal(md.includes('interfaceId'), false); +}); + +test('renderer: perLink shape — surfaces per-link warn/error icons', () => { + const results = [ + R('wanLatency', 'warn', '1 path in warning', null, { + total: 3, ok: 2, warn: 1, error: 0, + standardLabel: 'warn > 150ms, error > 400ms', + perLink: [ + { link: 'Inet1', value: 22, verdict: 'ok' }, + { link: 'Inet2', value: 175, verdict: 'warn' }, + { link: '5G-LTE', value: 450, verdict: 'error' }, + ], + }), + ]; + const md = renderVoiceDiagMarkdown(results, { storeNum: '782', detailed: true }); + assert.match(md, /✅ Inet1: 22/); + assert.match(md, /⚠️ Inet2: 175/); + assert.match(md, /❌ 5G-LTE: 450/); +}); + +test('renderer: link-state shape → up/down/unknown roll-up + offender list', () => { + const results = [ + R('wanLinkState', 'error', '1 WAN path down', null, { + total: 3, up: 2, down: 1, unknown: 0, + offenders: ['5G-LTE-00782'], + unknownLabels: [], + }), + ]; + const md = renderVoiceDiagMarkdown(results, { storeNum: '782', detailed: true }); + assert.match(md, /Roll-up: 3 total · 2 up · 1 down · 0 unknown/); + assert.match(md, /Down: 5G-LTE-00782/); + // No stringified offenders array + assert.equal(md.includes('"offenders"'), false); +}); + +test('renderer: alarm shape → counts one-liner + recent alarm list', () => { + const results = [ + R('wanAlarms', 'warn', '2 major alarms in last hour', null, { + critical: 0, major: 2, minor: 1, + recentSamples: [ + { type: 'NETWORK_ANYNETLINK_DOWN', severity: 'major' }, + { code: 'DEVICE_HB_MISSED', severity: 'minor' }, + ], + }), + ]; + const md = renderVoiceDiagMarkdown(results, { storeNum: '782', detailed: true }); + assert.match(md, /Counts: 🔴 0 critical · 🟠 2 major · 🟡 1 minor/); + assert.match(md, /Recent:/); + assert.match(md, /NETWORK_ANYNETLINK_DOWN \(major\)/); + assert.match(md, /DEVICE_HB_MISSED \(minor\)/); +}); + +test('renderer: healthscore shape → breakdown only (value already in message)', () => { + const results = [ + R('wanHealthscore', 'ok', 'Healthscore 100/100 (>=80).', null, { + value: 100, warnThresh: 80, errorThresh: 60, breakdown: {}, + }), + ]; + const md = renderVoiceDiagMarkdown(results, { storeNum: '782', detailed: true }); + // Breakdown is empty → no useless "- value: 100, warnThresh: 80..." line + assert.equal(md.includes('warnThresh'), false, 'threshold constants already in the message'); + assert.equal(md.includes('errorThresh'), false); +}); + +test('renderer: healthscore shape → surfaces breakdown when populated', () => { + const results = [ + R('wanHealthscore', 'warn', 'Score 75', null, { + value: 75, warnThresh: 80, errorThresh: 60, + breakdown: { link_health: 60, device_health: 85 }, + }), + ]; + const md = renderVoiceDiagMarkdown(results, { storeNum: '782', detailed: true }); + assert.match(md, /Breakdown: link_health: 60 · device_health: 85/); +}); + +test('renderer: WAN window banner shown when wanWindowMinutes provided AND a wan* check is present', () => { + const results = [ + R('wanHealthscore', 'ok', 'ok', null, { value: 100, warnThresh: 80, errorThresh: 60 }), + ]; + const md = renderVoiceDiagMarkdown(results, { + storeNum: '782', detailed: false, wanWindowMinutes: 1440, + }); + assert.match(md, /WAN window: 1d/); +}); + +test('renderer: WAN window banner hidden when no wan* check runs', () => { + const results = [R('dnd', 'ok', 'ok')]; + const md = renderVoiceDiagMarkdown(results, { + storeNum: '782', wanWindowMinutes: 60, + }); + assert.equal(md.includes('WAN window'), false, + 'no need to advertise a WAN window when no WAN check ran'); +}); + +test('renderer: WAN window banner formats: 15m / 1h / 6h / 1d', () => { + const wanResult = [ + R('wanLatency', 'ok', 'ok', null, { + total: 1, ok: 1, warn: 0, error: 0, + perLink: [{ link: 'A', value: 20, verdict: 'ok' }], + }), + ]; + const cases = [ + { min: 15, expect: 'WAN window: 15m' }, + { min: 60, expect: 'WAN window: 1h' }, + { min: 360, expect: 'WAN window: 6h' }, + { min: 1440, expect: 'WAN window: 1d' }, + { min: 45, expect: 'WAN window: 45m' }, + ]; + for (const c of cases) { + const md = renderVoiceDiagMarkdown(wanResult, { storeNum: '1', wanWindowMinutes: c.min }); + assert.ok(md.includes(c.expect), `${c.min} minutes → "${c.expect}", got:\n${md}`); + } +});