diff --git a/.env.example b/.env.example index 2de4eb9..f57a99c 100644 --- a/.env.example +++ b/.env.example @@ -122,6 +122,23 @@ WEBEX_TOKENS_PATH=./config/webex-service-tokens.json # PRISMA_PASSWORD= # PRISMA_LEGACY_BASE_URL=https://api.cloudgenix.com +# --- Prisma client-side concurrency cap --- +# Maximum number of in-flight Prisma requests at any moment. Prisma +# throttles bursts hard — /voicediag fans out 10+ metric calls per +# request, and without this cap they'd all fire simultaneously and +# 429-cascade. Default 3 is empirically safe on the observed tenant. +# Bump if your tenant tolerates more parallelism (fewer 429s), lower +# if you still see them. +# PRISMA_MAX_INFLIGHT=3 + +# --- Prisma Strata Cloud Manager (SCM) UI base URL --- +# Used only for building "View in Prisma UI" deep links surfaced in +# the /phonestatus WAN follow-up + /voicediag per-app details. The +# API endpoints continue to use PRISMA_SASE_BASE_URL. Override this +# only for tenants on a partner-branded or region-specific SCM +# domain. Default matches the production SCM hostname. +# PRISMA_UI_BASE_URL=https://stratacloudmanager.paloaltonetworks.com + # --- WAN voice-quality thresholds (used by /voicediag WAN bucket + the # /phonestatus WAN follow-up renderer) --- # Defaults follow ITU-T G.114 (latency), G.711 PLC (loss), and RFC 3550 @@ -138,6 +155,61 @@ WEBEX_TOKENS_PATH=./config/webex-service-tokens.json # WAN_STANDARD_HEALTHSCORE_WARN=80 # WAN_STANDARD_HEALTHSCORE_ERROR=60 +# --- Per-application voice-quality thresholds --- +# These grade the WORST-window value in the 24h series (rather than +# the average) because for voice a brief 15-min degradation is a bad +# operator experience even if the daily avg looks fine. Defaults are +# less strict than the link-probe thresholds above — otherwise a +# single 5-min bad window would drown the operator in warnings. +# WAN_STANDARD_APP_MOS_WARN=4.0 +# WAN_STANDARD_APP_MOS_ERROR=3.5 +# WAN_STANDARD_APP_LOSS_WARN_PCT=5 +# WAN_STANDARD_APP_LOSS_ERROR_PCT=15 +# WAN_STANDARD_APP_JITTER_WARN_MS=30 +# WAN_STANDARD_APP_JITTER_ERROR_MS=50 + +# --- Per-application DPI voice-metrics enablement --- +# When set, /phonestatus and /voicediag will fetch REAL voice-traffic +# quality metrics (MOS, packet loss, jitter, bandwidth) from Prisma's +# Application Path Details endpoint for the configured voice app. This +# is DPI data on actual RTP frames — the LQM link-probes above are +# synthetic probes across the WAN circuits and can look green while +# real audio is degrading (Prisma UI shows both; this env exposes the +# second one to the bot). +# +# Which app to point at is a tenant choice: +# - Webex_Calling_RTP ← recommended for Webex Calling shops. The +# Webex-specific DPI signature excludes noise +# from other UDP traffic and gives materially +# more accurate per-call quality signal. +# - rtp-base ← generic RTP DPI signature. Catches all RTP, +# including non-Webex flows; use if you have +# mixed voice apps or a tenant that doesn't +# break out per-provider signatures. +# - MS_Teams_RTP / Zoom_RTP / etc. — for those shops. +# +# To find the app id for your tenant: +# npm run prisma:probe -- appdefs webex # or `rtp`, `teams`, `zoom` +# or grab it from the Prisma UI's Application Path Details URL. Leave +# blank to disable the extra API calls entirely (checks return skipped +# with an explanation). +# PRISMA_APP_ID_VOICE= + +# Optional human-friendly display label for the app configured above. +# Shown in the "Voice Traffic Quality (…)" section headers and check +# messages. Defaults to "voice" if unset. Recommend setting it to the +# exact Prisma UI app label so operators can cross-reference: +# PRISMA_APP_NAME_VOICE=Webex_Calling_RTP + +# --- BACKWARDS-COMPAT: PRISMA_APP_ID_RTP_BASE (deprecated) --- +# Original name from the initial rtp-base-only release. Still honored +# if set AND PRISMA_APP_ID_VOICE is unset, but logs a one-time +# deprecation warning on first use. When falling back to this var, the +# display name defaults to "rtp-base" so existing dashboards keep +# reading the same label. Rename to PRISMA_APP_ID_VOICE at your +# convenience — the old var will be removed in a future release. +# PRISMA_APP_ID_RTP_BASE= + # --- WAN bucket global kill-switch (mirror of VOICE_STANDARD_ENABLED for # the port bucket). Set to `false` to silence the entire WAN check # bucket while a Prisma cleanup is in progress. The /phonestatus @@ -147,12 +219,31 @@ WEBEX_TOKENS_PATH=./config/webex-service-tokens.json # --- WAN look-back window (minutes) used when the operator doesn't # pass `--window` on /voicediag (and always used by /phonestatus). -# Applies to healthscore + LQM. Alarms window is -# max(60, WAN_STANDARD_WINDOW_MINUTES) since sub-hour alarm queries -# are usually too noisy to be actionable. Prisma's finest LQM -# bucket is 5m; anything <= 5 is snapped up. -# Common values: 15 (real-time), 60, 360 (6h), 1440 (24h — default). -# WAN_STANDARD_WINDOW_MINUTES=1440 +# Applies to healthscore + LQM + per-app voice DPI + alarms. +# Alarms window is max(60, WAN_STANDARD_WINDOW_MINUTES) since +# sub-hour alarm queries are usually too noisy to be actionable. +# +# Default: 10080 (7 days). Widened from 24h because per-app DPI +# metrics (Webex_Calling_RTP etc.) only generate datapoints WHEN +# CALLS HAPPEN — a store that takes 3-4 Webex calls per day gives +# only ~30 samples in 24h, well below the point where "worst- +# window" statistics mean anything. Seven days consistently +# yields 150-300+ per-app samples across stores of any size. +# +# For live-incident triage where you want a fresh snapshot, pass +# `--window 1h` or `--window 24h` on /voicediag (or lower this +# global default here). +# +# Interval selection is automatic per window size — see +# integrations/paloalto/metrics.js:pickAppMetricInterval / +# pickInterval. Hard-capped at 10080 (7d); beyond that Prisma +# downsamples to 1-day buckets and the worst-window signal +# dissolves. If you truly need 30d, edit MAX_WINDOW_MINUTES in +# services/enrichment/sdwanEnrichment.js and confirm the interval +# pickers still yield useful granularity for the check semantics. +# +# Common values: 60 (1h — live triage), 1440 (24h), 10080 (7d — default). +# WAN_STANDARD_WINDOW_MINUTES=10080 # ----------------------------------------------------------------------------- # /voicediag — Store voice-line standards diff --git a/commands/help.js b/commands/help.js index 4482d2a..6d1cddf 100644 --- a/commands/help.js +++ b/commands/help.js @@ -18,7 +18,7 @@ const SHORT_HELP = { // AV / phones avstatus: 'AV / device status for a store (alias: /wostatus)', - phonestatus: 'DECT + IP phone status for a store (plus 24h WAN follow-up)', + phonestatus: 'DECT + IP phone status for a store (plus 7d WAN follow-up)', voicediag: 'Deep voice diagnostic: Webex Calling features + SD-WAN quality with fix cards', // Jira @@ -57,7 +57,8 @@ const LONG_HELP = { notes: [ 'Shows DECT basestations + IP phones with Meraki links.', 'Detailed mode adds firmware, serial, SIP details and errors.', - 'When the store is Prisma SD-WAN managed (site name `CG` padded to 5 digits), a follow-up **WAN Diagnostics** message arrives with per-path latency/jitter/loss/MOS, site healthscore, and any alarms — averaged over the last 24 hours (override via `WAN_STANDARD_WINDOW_MINUTES`).', + 'When the store is Prisma SD-WAN managed (site name `CG` padded to 5 digits), a follow-up **WAN Diagnostics** message arrives with per-path latency/jitter/loss/MOS, site healthscore, and any alarms — averaged over the last 7 days by default (widened from 24h so sporadic Webex Calling stores get enough call samples; override via `WAN_STANDARD_WINDOW_MINUTES` or pass `--window 24h` on /voicediag).', + 'If `PRISMA_APP_ID_VOICE` is configured (e.g. pointing at `Webex_Calling_RTP` for Webex Calling shops), the follow-up also includes a **Voice Traffic Quality** section with real DPI-measured MOS / packet loss / jitter for that app — surfaces transient degradation the 7d link-probe averages smooth away.', 'For an in-depth voice diagnostic with per-user Webex Calling checks + fix cards, use `/voicediag `.', 'Web dashboard: `/phone-store-dashboard.html`.', ], @@ -74,18 +75,20 @@ const LONG_HELP = { examples: [ '/voicediag 782', '/voicediag 782 detail', - '/voicediag 782 --window 15m', - '/voicediag 782 --window 6h --only wanLatency,wanJitter,wanLoss,wanMos', + '/voicediag 782 --window 24h', + '/voicediag 782 --window 15m --only wanLatency,wanJitter,wanLoss,wanMos', + '/voicediag 782 --only wanAppRtpMos,wanAppRtpLoss,wanAppRtpJitter', '/voicediag 782 --only dnd,callForwarding,voicemail', '/voicediag list-checks', ], notes: [ - 'Runs a full battery of per-user Webex Calling checks (DND, call forwarding, voicemail, call intercept, call waiting, outgoing permission, etc.) plus eight SD-WAN checks (site, healthscore, link state, latency, jitter, loss, MOS, alarms) against the store\'s Prisma tenant.', + 'Runs a full battery of per-user Webex Calling checks (DND, call forwarding, voicemail, call intercept, call waiting, outgoing permission, etc.) plus eleven SD-WAN checks (site, healthscore, link state, latency, jitter, loss, MOS, per-app voice MOS/loss/jitter, alarms) against the store\'s Prisma tenant.', + 'The per-app checks measure REAL voice-traffic quality via Prisma DPI (worst 5-minute window over the configured look-back, default 7d), which catches transient degradation the link-probe averages smooth away. Feature-gated on `PRISMA_APP_ID_VOICE` env (set to a Prisma app id like `Webex_Calling_RTP` or `rtp-base`) — checks return skipped with an explanation when not configured.', 'Default view hides OK checks and highlights errors/warnings/skipped. Pass `detail` (or `detailed`) to also see OK checks with expanded per-link tables + thresholds + roll-ups.', 'Fixable issues (e.g. DND on, forwarding to wrong number) post a per-issue confirmation card. A single "apply all" card lets you fix everything at once after reviewing.', '`--only` restricts the run to specific check ids (comma-separated). Use `/voicediag list-checks` to see every registered id + its scope.', - '`--window` overrides the WAN look-back window (accepts `15m`, `1h`, `6h`, `24h`, `1d`, or a bare minute count). Applies to healthscore + LQM (latency/jitter/loss/MOS) fetches. Alarms are floored at 60m. Default is 24h — set `WAN_STANDARD_WINDOW_MINUTES` for a different global default.', - 'Voice-quality thresholds default to ITU-T G.114 / RFC 3550. Override any of them via `WAN_STANDARD_*` env vars (see `.env.example`). Kill-switch: `WAN_STANDARD_ENABLED=false` silences the whole WAN bucket.', + '`--window` overrides the WAN look-back window (accepts `15m`, `1h`, `6h`, `24h`, `1d`, `7d`, or a bare minute count; hard-capped at 7d). Applies to healthscore + LQM (latency/jitter/loss/MOS) + per-app voice fetches + alarms. Alarms are floored at 60m. Default is 7d — pass `--window 24h` for a tighter view during live-incident triage, or set `WAN_STANDARD_WINDOW_MINUTES` for a different global default.', + 'Voice-quality thresholds default to ITU-T G.114 / RFC 3550. Override any of them via `WAN_STANDARD_*` env vars (see `.env.example`). Per-app thresholds live under `WAN_STANDARD_APP_*`. Kill-switch: `WAN_STANDARD_ENABLED=false` silences the whole WAN bucket.', 'HTTP equivalent: `?storeNum=[&detailed=true][&only=id1,id2][&window=15m]`. HTTP callers see the markdown snapshot only — cards are chat-only.', 'Audit log: every remediation apply/cancel is logged under `voicediag:audit` with the requester identity.', ], diff --git a/commands/voiceDiag.js b/commands/voiceDiag.js index da76e1a..ad78aad 100644 --- a/commands/voiceDiag.js +++ b/commands/voiceDiag.js @@ -76,14 +76,17 @@ export async function handleVoiceDiag(bot, trigger) { .filter(Boolean); // Look-back window for WAN checks (healthscore + LQM + alarms). - // Accepts `15m`, `1h`, `24h`, `1d` etc. Defaults to whatever - // `buildContext` resolves via env WAN_STANDARD_WINDOW_MINUTES. + // Accepts `15m`, `1h`, `24h`, `1d`, `7d` etc. Defaults to whatever + // `buildContext` resolves via env WAN_STANDARD_WINDOW_MINUTES + // (currently 7d — anything past that gets silently clamped by + // normalizeWindowMinutes since Prisma downsamples too coarsely + // beyond that to preserve worst-window signal). const windowRaw = pickWindowArg(args) ?? query.window ?? null; const windowMinutes = parseWindowMinutes(windowRaw); if (windowRaw && windowMinutes == null) { await bot.say( 'markdown', - `Unrecognised window \`${windowRaw}\`. Use \`15m\`, \`1h\`, \`6h\`, \`24h\`, or \`1d\`.`, + `Unrecognised window \`${windowRaw}\`. Use \`15m\`, \`1h\`, \`6h\`, \`24h\`, \`1d\`, or \`7d\`.`, ); return; } diff --git a/integrations/paloalto/client.js b/integrations/paloalto/client.js index 12be373..f731038 100644 --- a/integrations/paloalto/client.js +++ b/integrations/paloalto/client.js @@ -92,6 +92,56 @@ export const paloAltoAxios = axios.create({ }, }); +// ─── Global concurrency limiter ───────────────────────────────────── +// +// Prisma throttles bursts hard — fan-out patterns like /voicediag +// (which fires 10+ parallel metric calls per request) can trip the +// tenant rate limit and cause a cascade where retries compete with +// still-pending original calls, wasting the whole batch. Cap the +// in-flight count with a small semaphore so bursts are naturally +// serialized instead. Empirically 3 works on the observed tenant — +// bump `PRISMA_MAX_INFLIGHT` in the env if the ceiling is looser. +// +// The permit is acquired in the request interceptor and released in +// BOTH response paths (success + error) so retries don't hold the +// permit through the backoff window (that would let 429'd requests +// stall queued fresh requests). 429 retries reacquire on their way +// back through the request interceptor. +const MAX_INFLIGHT = Math.max(1, Number(process.env.PRISMA_MAX_INFLIGHT) || 3); +let inflightCount = 0; +const inflightWaiters = []; + +function acquirePermit() { + if (inflightCount < MAX_INFLIGHT) { + inflightCount++; + return Promise.resolve(); + } + return new Promise((resolve) => inflightWaiters.push(resolve)); +} + +function releasePermit() { + const next = inflightWaiters.shift(); + if (next) { + // Hand permit directly to the next waiter — don't decrement + + // increment (that's a race window in case a new caller arrives + // between the two ops). + next(); + } else { + inflightCount = Math.max(0, inflightCount - 1); + } +} + +/** @internal Test-only: reset the semaphore between tests. */ +export function _resetPrismaConcurrency() { + inflightCount = 0; + inflightWaiters.length = 0; +} + +/** @internal Test-only: peek at the current in-flight count. */ +export function _prismaInflightCount() { + return inflightCount; +} + // Request interceptor: (a) apply resolved baseURL on the fly if the // caller passed a relative URL; (b) inject the correct auth header // for the current auth mode; (c) prime the SASE unified SD-WAN @@ -128,6 +178,21 @@ paloAltoAxios.interceptors.request.use(async (cfg) => { } } + // Concurrency-cap acquire. This is the last step of the request + // interceptor so all pre-request work (baseURL, auth, session + // priming) happens WITHOUT holding a permit — otherwise a slow + // priming call would eat one of the three inflight slots for its + // whole duration and shrink the effective ceiling. + // + // `_permitAcquired` is a per-request flag consumed by the response + // interceptor. Priming (a nested paloAltoAxios call) is + // intentionally exempt via `_skipConcurrencyGate` so we don't + // deadlock the last permit waiting for its own dependency. + if (!cfg._skipConcurrencyGate) { + await acquirePermit(); + cfg._permitAcquired = true; + } + logger('paloalto:request', `${cfg.method?.toUpperCase()} ${cfg.baseURL || ''}${cfg.url}`, 'debug'); return cfg; }); @@ -142,10 +207,27 @@ const MAX_429_RETRIES = 2; const BASE_429_BACKOFF_MS = 1500; paloAltoAxios.interceptors.response.use( - (response) => response, + (response) => { + // Success path: release the concurrency permit acquired in the + // request interceptor before returning to the caller. + if (response.config?._permitAcquired) { + response.config._permitAcquired = false; + releasePermit(); + } + return response; + }, async (error) => { const status = error.response?.status; + // Release the concurrency permit BEFORE any retry (401/429). The + // retry re-enters the request interceptor and will reacquire — + // holding the permit across the backoff would let a single + // throttled request block fresh callers. + if (error.config?._permitAcquired) { + error.config._permitAcquired = false; + releasePermit(); + } + if (status === 401 && !error.config?._retry) { logger('paloalto:client', '401 → forcing token refresh + one-shot retry', 'warn'); try { diff --git a/integrations/paloalto/index.js b/integrations/paloalto/index.js index 9a95563..675fce5 100644 --- a/integrations/paloalto/index.js +++ b/integrations/paloalto/index.js @@ -21,9 +21,16 @@ export { export { LQM_METRIC_NAMES, + APP_METRIC_NAMES, getHealthscore, getLqmMetric, + getAppMetric, getAlarms, } from './metrics.js'; +export { + getPrismaUiBaseUrl, + buildAppDetailsUrl, +} from './urls.js'; + export { default } from './client.js'; diff --git a/integrations/paloalto/metrics.js b/integrations/paloalto/metrics.js index 63e4d83..a8a22bc 100644 --- a/integrations/paloalto/metrics.js +++ b/integrations/paloalto/metrics.js @@ -145,6 +145,11 @@ function nowIso() { // Prisma's valid `interval` enum (verified via HTTP 400 error message). // `15min`, `30min` etc. are NOT accepted — snap up to the next valid // bucket (1hour) rather than silently down-sampling. +// +// Used for LQM and healthscore. Those endpoints return a SINGLE +// aggregated value per path/site (regardless of interval), so the +// interval only affects Prisma's internal downsampling — coarser is +// fine, and 1day for a 24h window is the cheapest ask. function pickInterval(minutes) { if (minutes <= 1) return '1min'; if (minutes <= 5) return '5min'; @@ -153,6 +158,27 @@ function pickInterval(minutes) { return '1day'; } +// App-metric variant. UNLIKE LQM, per-app metrics return the FULL +// datapoint time series (metrics[].series[].data[].datapoints[]) and +// downstream code aggregates {avg, min, max, p95} client-side. That +// only works if the interval is fine enough to capture the transient +// windows we're trying to surface. HAR (2026-07-09) confirms the +// Prisma UI queries `5min` for a 24h window (yielding 288 datapoints +// per metric) — matching that keeps our worst-window statistics +// meaningful. Falling back to `1day` (as `pickInterval` does) would +// collapse the whole series into 1-2 aggregated points and hide the +// exact degradation these checks exist to catch. +function pickAppMetricInterval(minutes) { + if (minutes <= 1) return '1min'; + if (minutes <= 5) return '5min'; + if (minutes <= 60) return '5min'; // 12 points per hour + if (minutes <= 1440) return '5min'; // 24h → 288 points (HAR-confirmed) + // Beyond 24h, cap the point count with hourly buckets: 7d @ 1h = 168 + // points which is still a manageable payload. + if (minutes <= 10080) return '1hour'; + return '1day'; +} + // Small helper that logs a failed metric call with a response-body // preview AND (on 4xx) the request body that Prisma rejected — the // most common failure mode is a metric name or filter key drifting @@ -291,6 +317,146 @@ export async function getLqmMetric(siteId, waninterfaceIds, metricKey, windowMin } } +// ────────────────────────────────────────────── +// Per-application "Application Path Details" metrics (v2.6 API) +// ────────────────────────────────────────────── +// +// These are what Prisma's Application Path Details dashboard renders: +// real audio/video traffic quality per app (rtp-base, +// Webex_Calling_RTP, MS_Teams_RTP, etc.) — measured on ACTUAL user +// packets via DPI, not on synthetic LQM probes across the underlying +// link. For voice diagnostics this is +// the more meaningful signal — link probes can pass while real RTP +// experiences packet loss spikes and MOS dips during flap events +// that get averaged out of the link probe view. +// +// Registry entries were confirmed via HAR captures 2026-07-09 against +// a live tenant (site CG00127) for both `rtp-base` (id 15932000365560116) +// and `Webex_Calling_RTP` (id 1708539371717015196). The metric names, +// units, and filter shapes are identical across voice apps — the only +// per-app knob is filter.app, so the check code is app-agnostic. The +// winning shapes are shown per-metric in the table below. +// +// Key filter-shape quirks: +// - `filter.direction` is REQUIRED as `"Ingress"` for audio-quality +// metrics (loss/jitter/MOS are what you RECEIVE). Bandwidth is +// bidirectional aggregate and does NOT take a direction. +// - `filter.path_type` is REQUIRED for AppPerf* metrics (loss, +// jitter, bandwidth) but NOT for AppAudioMos. +// - `filter.app` uses the numeric app id (resolvable via +// `POST /sdwan/v2.5/api/appdefs/query`), NOT the display name. +// +// Response shape is IDENTICAL to v2.6 healthscore +// (metrics[].series[].data[].datapoints[{time, value}]) so downstream +// parsing can reuse the same helper. + +const ALL_PATH_TYPES = Object.freeze( + ['DirectInternet', 'VPN', 'PrivateWAN', 'PrivateVPN', 'ServiceLink'], +); + +// eslint-disable-next-line no-restricted-syntax -- frozen shared registry. +export const APP_METRIC_NAMES = Object.freeze({ + mos: { + name: 'AppAudioMos', + unit: 'count', + direction: 'Ingress', + includePathType: false, + // Lower is worse for MOS: 5=excellent, 4=good, 3.5=acceptable, + // <3=impaired, <2=unintelligible. Grade against worst-window + // rather than average — a 24h avg smooths over the flap events + // that actually degrade calls. + lowIsBad: true, + }, + loss: { + name: 'AppPerfUDPAudioPacketLoss', + unit: 'percentage', + direction: 'Ingress', + includePathType: true, + lowIsBad: false, + }, + jitter: { + name: 'AppPerfUDPAudioJitter', + unit: 'milliseconds', + direction: 'Ingress', + includePathType: true, + lowIsBad: false, + }, + bandwidth: { + name: 'AppPerfUDPAudioBandwidth', + unit: 'Mbps', + direction: null, + includePathType: true, + lowIsBad: null, // bandwidth isn't a quality metric — used for context only + }, +}); + +// v2.6 monitor/metrics path (SASE unified prefix vs. legacy). +function appMetricsPath() { + return isSase() + ? '/sdwan/monitor/v2.6/api/monitor/metrics' + : '/v2.6/api/monitor/metrics'; +} + +/** + * Fetch a single per-app metric time series for a site. + * + * Returns the raw Prisma response body (with metrics[].series[].data + * [].datapoints[] intact) or null on failure. The datapoints array is + * usually 24h × 5min = 288 points; the caller is responsible for + * client-side aggregation (avg / min / max / p95) because for voice + * quality the WORST window matters more than the average. + * + * @param {string} siteId + * @param {string} appId Prisma numeric app id (NOT display name) + * @param {'mos'|'loss'|'jitter'|'bandwidth'} metricKey + * @param {number} windowMinutes default 1440 (24h) — matches the + * shipping default for WAN checks + * @returns {Promise} + */ +export async function getAppMetric(siteId, appId, metricKey, windowMinutes = 1440) { + const spec = APP_METRIC_NAMES[metricKey]; + if (!spec) { + throw new Error( + `Unknown app metric key "${metricKey}" ` + + `(expected: ${Object.keys(APP_METRIC_NAMES).join(', ')})`, + ); + } + if (!siteId || !appId) { + logger('paloalto:metrics', + `getAppMetric(${metricKey}): missing siteId or appId — skipping`, 'debug'); + return null; + } + + const filter = { + site: [String(siteId)], + app: [String(appId)], + }; + if (spec.includePathType) filter.path_type = [...ALL_PATH_TYPES]; + if (spec.direction) filter.direction = spec.direction; + + try { + const res = await paloAltoAxios.post(appMetricsPath(), { + start_time: windowStart(windowMinutes), + end_time: nowIso(), + // NOT pickInterval — see pickAppMetricInterval doc for why: + // app metrics need the fine-grained series or the client-side + // worst-window aggregation is meaningless. + interval: pickAppMetricInterval(windowMinutes), + metrics: [{ + name: spec.name, + statistics: ['average'], + unit: spec.unit, + }], + filter, + view: {}, + }); + return res.data || null; + } catch (err) { + logMetricFailure(`getAppMetric(${siteId}, app=${appId}, ${metricKey})`, err); + return null; + } +} + // ────────────────────────────────────────────── // Alarms (via the events/query endpoint) // ────────────────────────────────────────────── diff --git a/integrations/paloalto/urls.js b/integrations/paloalto/urls.js new file mode 100644 index 0000000..fccd96c --- /dev/null +++ b/integrations/paloalto/urls.js @@ -0,0 +1,52 @@ +// src/integrations/paloalto/urls.js +// +// Prisma Strata Cloud Manager (UI) URL builders. Lives beside the +// API client so any new deep-link pattern we discover has one +// obvious home and one obvious set of tests. +// +// Kept intentionally tiny: no HTTP, no async, no fancy templating — +// each function is pure ("give me the URL for X"). Consumers should +// pass ids they already have from the API (site.id, appAudio.appId, +// etc.); nothing in here should do its own API lookups. + +const PRISMA_UI_DEFAULT_BASE = 'https://stratacloudmanager.paloaltonetworks.com'; + +/** + * Resolve the Strata Cloud Manager base URL. + * + * Defaults to the production hostname. Overridable via + * `PRISMA_UI_BASE_URL` for tenants using a partner-branded or + * region-specific SCM domain, and for tests. Read at call time so + * tests can toggle without a module cache reset. + * + * The trailing slash is normalized off so all builders below can + * safely concatenate `${base}/path/…`. + */ +export function getPrismaUiBaseUrl() { + const raw = process.env.PRISMA_UI_BASE_URL; + const base = raw && raw.trim() ? raw.trim() : PRISMA_UI_DEFAULT_BASE; + return base.replace(/\/+$/, ''); +} + +/** + * Deep link into the SCM "Application Path Details" view for a + * single app + site combination. This is the exact page the Prisma + * UI opens when the operator clicks an app on the site's dashboard — + * the RTP/MOS/loss/jitter charts we scrape via `getAppMetric()`. + * + * Sample: + * https://stratacloudmanager.paloaltonetworks.com/insights/ + * operational/sdwan-applications/{appId}/site/{siteId}/details + * + * Returns null when either id is missing so callers can no-op + * without a null guard on every render path. + * + * @param {string|number} siteId Numeric Prisma site id + * @param {string|number} appId Numeric Prisma app id + * @returns {string|null} + */ +export function buildAppDetailsUrl(siteId, appId) { + if (!siteId || !appId) return null; + const base = getPrismaUiBaseUrl(); + return `${base}/insights/operational/sdwan-applications/${appId}/site/${siteId}/details`; +} diff --git a/scripts/prismaProbe.js b/scripts/prismaProbe.js index af312b1..bbe12c4 100644 --- a/scripts/prismaProbe.js +++ b/scripts/prismaProbe.js @@ -69,6 +69,27 @@ * the metric's identifier is wrong (400 METRIC_UNIT_NOT_SUPPORTED * or METRIC_NOT_FOUND). * + * try-shapes app-list + * Discover which per-app metrics Prisma exposes for the site + * (Webex_Calling_RTP, rtp-base, MS_Teams_RTP, voice_rtp, ...). + * Run this FIRST before any of the app-audio-* probes so you + * know the right app id for your tenant. + * + * try-shapes app-audio-mos + * try-shapes app-audio-loss + * try-shapes app-audio-jitter + * try-shapes app-audio-bandwidth + * Sweep candidate metric name × unit combinations for the + * "Application Path Details" per-app DPI metrics. These are + * what actual voice traffic (e.g. Webex_Calling_RTP, rtp-base) + * experiences, which is often materially worse than the LQM + * link-probe signal. Uses + * the HAR-confirmed shape: filter.app=[] (numeric id, NOT + * display name), view.individual="app", no filter.site. Run + * `appdefs ` first to look up the app id. + * Once a shape lands, feed the winner back into + * integrations/paloalto/metrics.js as a getAppMetric(). + * * Global flags: * --json Emit JSON output instead of pretty-printed. * --show-body Print the request body on 2xx as well as 4xx. @@ -352,6 +373,54 @@ async function cmdRaw(args) { printVerdict(verdict, args.flags); } +/** + * appdefs — resolve the tenant's app catalog. Optional filter arg: + * - `appdefs` → dump all apps (first `--limit` rows, + * default 500) + * - `appdefs ` → grep the `display_name` and `name` + * fields for a case-insensitive match + * + * Uses the endpoint confirmed working on this tenant via app-list probe: + * POST /sdwan/v2.5/api/appdefs/query + * + * Response items each have `id`, `name`, `display_name`, `category`. + * The `id` is what per-app metric calls need in `filter.app`. + */ +async function cmdAppdefs(args) { + const needle = args._[1] || ''; + const limit = Number(args.flags.limit) || 500; + heading(`appdefs — limit=${limit}${needle ? `, needle="${needle}"` : ''}`); + const verdict = await fire({ + method: 'POST', + url: '/sdwan/v2.5/api/appdefs/query', + body: { limit }, + }); + if (!verdict.ok) { + printVerdict(verdict, args.flags); + return; + } + const items = verdict.data?.items || []; + info(`fetched ${items.length} appdefs (total_count=${verdict.data?.total_count || '?'})`); + const rows = needle + ? items.filter((a) => { + const hay = [ + a?.name, a?.display_name, a?.category, a?.id, + ].filter(Boolean).join(' ').toLowerCase(); + return hay.includes(needle.toLowerCase()); + }) + : items; + ok(`${rows.length} match${rows.length === 1 ? '' : 'es'}`); + console.log('id name display_name'); + console.log('──────────────────────────── ─────────────────────────── ─────────────────────'); + for (const a of rows.slice(0, 50)) { + const id = String(a?.id || '').padEnd(28); + const name = String(a?.name || '').padEnd(27); + const disp = String(a?.display_name || ''); + console.log(`${id} ${name} ${disp}`); + } + if (rows.length > 50) info(`… +${rows.length - 50} more (use --limit to widen the initial fetch, or narrow the needle)`); +} + // ─── try-shapes: combinatorial schema testing ─────────────────────── function pickInterval5min() { return '5min'; } @@ -605,7 +674,298 @@ async function cmdTryShapes(args) { return; } - throw new Error(`unknown try-shapes target "${which}" — expected "health", "lqm", "lqm-latency", "lqm-jitter", "lqm-loss", or "lqm-mos"`); + // ── App-metrics matrix ──────────────────────────────────────────── + // Explore Application Path Details signals — the "rtp-base" style + // per-app DPI metrics that the Prisma UI's Application Path Details + // page renders. These are what actual voice traffic experiences, + // as opposed to LQM which measures synthetic link probes. + // + // Unlike LQM, we don't yet know the metric name/unit or the exact + // filter shape (specifically, whether Prisma expects `app`, `apps`, + // `app_name`, or `application`). So each candidate combines: + // - endpoint (v2.6 monitor/metrics is the modern path; v2.0 has + // an appstats + aggregate_flows variant we also try) + // - metric name variants Prisma might use + // - unit variants + // - filter-key variants for the app selector + // + // Discovery flow: run `try-shapes app-list ` first to see + // what app names Prisma actually has for the site (rtp-base vs + // rtp vs voice_rtp vs ...). Then feed the right one into + // `try-shapes app-audio-mos|loss|jitter`. + // NOTE: HAR capture 2026-07-09 confirmed Prisma's v2.6 per-app family + // uses plain Title-case names (NO `PointMetric` suffix). Real names + // observed: ApplicationHealthscore/gauge, BandwidthUsage/Mbps, + // TCPFlowCount/count. The audio metric names below all follow the + // same convention — no `PointMetric`, no `Point`, just words. + // + // Also: filter uses `filter.app` with the APP ID (e.g. "15932...") + // NOT the display name ("rtp-base"). Look up the id with the + // `appdefs` subcommand first. + // Metric name × unit sweeps for the "Application Path Details" + // per-app metrics. Winners (confirmed via HAR 2026-07-09) are marked. + // The rejected guesses stay in as regression guards — if Prisma + // ever renames them we'll see it here. + const APP_METRIC_MATRIX = { + 'app-audio-mos': { + label: 'per-app audio MOS score', + // WINNER: AppAudioMos + count. NOTE: this metric requires + // filter.direction="Ingress" (audio quality is what you + // RECEIVE) and does NOT use filter.path_type. + metricNames: [ + 'AppAudioMos', // ← LIVE WINNER (2026-07-09) + 'AudioMOSScore', 'AudioMosScore', 'AudioMOS', 'AudioMos', + 'ApplicationAudioMOS', 'MOSScore', 'VoiceMOS', 'VoiceMOSScore', + ], + units: ['count', 'gauge', 'score'], + direction: 'Ingress', + includePathType: false, + }, + 'app-audio-loss': { + label: 'per-app audio packet loss', + // WINNER: AppPerfUDPAudioPacketLoss + percentage. Uses + // filter.direction="Ingress" AND filter.path_type=[all]. + metricNames: [ + 'AppPerfUDPAudioPacketLoss', // ← LIVE WINNER (2026-07-09) + 'AudioPacketLoss', 'AudioPktLoss', 'AudioLoss', + 'ApplicationAudioPacketLoss', 'PacketLoss', 'VoicePacketLoss', + ], + units: ['percentage', 'percent', 'gauge', 'count'], + direction: 'Ingress', + includePathType: true, + }, + 'app-audio-jitter': { + label: 'per-app audio jitter', + // WINNER: AppPerfUDPAudioJitter + milliseconds. Uses + // filter.direction="Ingress" AND filter.path_type=[all]. + metricNames: [ + 'AppPerfUDPAudioJitter', // ← LIVE WINNER (2026-07-09) + 'AudioJitter', 'ApplicationAudioJitter', 'Jitter', 'VoiceJitter', + ], + units: ['milliseconds', 'gauge', 'count'], + direction: 'Ingress', + includePathType: true, + }, + 'app-audio-bandwidth': { + label: 'per-app audio bandwidth', + // WINNER: AppPerfUDPAudioBandwidth + Mbps. Uses filter.path_type + // but does NOT need filter.direction (bandwidth is bidirectional + // aggregate on this endpoint). + metricNames: [ + 'AppPerfUDPAudioBandwidth', // ← LIVE WINNER (2026-07-09) + 'AudioBandwidth', 'AudioBandwidthUsage', 'ApplicationAudioBandwidth', + 'BandwidthUsage', // Tenant-wide bandwidth (also works, + // useful control that filter.app narrows it) + ], + units: ['Mbps', 'kbps', 'gauge'], + direction: null, + includePathType: true, + }, + }; + + if (APP_METRIC_MATRIX[which]) { + // Second positional arg is the APP ID (not the name) — Prisma's + // filter.app expects the numeric id. Run `appdefs ` to + // discover an app id (or grab one from the browser dev tools on + // the Prisma UI's Application Path Details page). + const appId = args._[3]; + if (!appId) { + throw new Error( + `usage: try-shapes ${which} \n` + + ` Use \`node scripts/prismaProbe.js appdefs \` to find the app id.`, + ); + } + heading(`try-shapes ${which} → siteId=${siteId} appId=${appId}`); + const target = APP_METRIC_MATRIX[which]; + + // HAR-confirmed request shape (rtp-base-metricCG00127.har, + // 2026-07-09): start_time+end_time required, filter.site works, + // audio-quality metrics need filter.direction="Ingress", + // AppPerf* metrics take filter.path_type, AppAudioMos does not, + // view is empty `{}` for these single-metric queries. + const endTime = new Date().toISOString(); + const startTime24h = new Date(Date.now() - 24 * 60 * 60 * 1000).toISOString(); + const ALL_PATH_TYPES = ['DirectInternet', 'VPN', 'PrivateWAN', 'PrivateVPN', 'ServiceLink']; + + const candidates = target.metricNames.flatMap((name) => + target.units.map((unit) => { + const filter = { site: [String(siteId)], app: [String(appId)] }; + if (target.includePathType) filter.path_type = ALL_PATH_TYPES; + if (target.direction) filter.direction = target.direction; + const filterHint = [ + `filter.app=[${appId}]`, + `filter.site=[${siteId}]`, + target.includePathType ? 'filter.path_type=[…]' : '', + target.direction ? `filter.direction="${target.direction}"` : '', + ].filter(Boolean).join(', '); + return { + label: `name="${name}", unit="${unit}", ${filterHint}`, + body: { + start_time: startTime24h, + end_time: endTime, + interval: pickInterval5min(), + metrics: [{ name, statistics: ['average'], unit }], + filter, + view: {}, + }, + }; + }), + ); + + const results = await runCandidates( + 'POST', + '/sdwan/monitor/v2.6/api/monitor/metrics', + candidates, {}, args.flags, + ); + printSummary(`${which} (${target.label})`, results); + return; + } + + // ── App discovery — list which app names Prisma sees at the site + // + // Two-pronged approach: + // 1. Probe several plausible appdefs / appdef-query endpoints so + // we can dump a global catalog of app names Prisma knows. + // 2. Sweep a large-ish list of candidate metric names against the + // v2.6 monitor/metrics endpoint with `view.individual="app"`. + // Any metric that returns 200 + a data payload will surface + // the app names as a side-effect (the response includes an + // `app` dimension per row). + // + // The correct v2.6 metric names for per-app / per-app-path signals + // aren't documented consistently — Prisma renames them across + // versions. This sweep should reveal at least one that works so we + // can then use it as the base for per-app-audio-* probes. + // + // As a fast alternative to blind guessing, the operator can also + // open the "Application Path Details" page in the Prisma UI with + // browser dev tools open (Network tab, filter for /monitor/) — the + // page fires the exact API call we need to mimic. Copy the request + // body from there into the raw subcommand for an instant answer. + if (which === 'app-list') { + heading(`try-shapes app-list → siteId=${siteId}`); + + // 1. Appdef catalog endpoints — different Prisma versions expose + // the list of known apps at different paths. All are read-only. + const appdefEndpoints = [ + { method: 'POST', url: '/sdwan/appdefs/v2.5/api/appdefs/query', + body: { limit: 200 }, + label: 'POST /appdefs/v2.5/api/appdefs/query {limit:200}' }, + { method: 'POST', url: '/sdwan/appdefs/v2.1/api/appdefs/query', + body: { limit: 200 }, + label: 'POST /appdefs/v2.1/api/appdefs/query {limit:200}' }, + { method: 'GET', url: '/sdwan/v2.5/api/appdefs', + label: 'GET /v2.5/api/appdefs (tenant-wide, top-level)' }, + { method: 'GET', url: '/sdwan/v2.1/api/appdefs', + label: 'GET /v2.1/api/appdefs (tenant-wide, top-level)' }, + { method: 'POST', url: '/sdwan/v2.5/api/appdefs/query', + body: { limit: 200 }, + label: 'POST /v2.5/api/appdefs/query {limit:200}' }, + { method: 'GET', url: '/sdwan/config/v2.5/api/appdefs', + label: 'GET /config/v2.5/api/appdefs' }, + ]; + + // 2. Metric-name catalog sweep. Confirmed via HAR capture 2026-07-09 + // that Prisma's v2.6 per-app family does NOT use the `PointMetric` + // suffix seen in the LQM family. Real names are plain Title case + // like ApplicationHealthscore, BandwidthUsage, TCPFlowCount. + // Below is a mix of KNOWN-WORKING names (as regression guards + + // dimension probes) and PLAUSIBLE audio-metric guesses following + // the observed naming convention. + const perAppMetricNames = [ + // ─── CONFIRMED WORKING NAMES (HAR 2026-07-09) ───────────────── + // Use these as canary checks — if a Prisma tenant update ever + // renames them, we'll see it here. + 'ApplicationHealthscore', + 'BandwidthUsage', + 'TCPFlowCount', + 'UDPFlowCount', + 'AppSuccessfulConnections', + 'AppSuccessfulTransactions', + 'AppFailedToEstablish', + 'AppTransactionFailures', + // ─── PLAUSIBLE AUDIO / MOS GUESSES (per naming convention) ──── + // Prisma UI shows Audio MOS Score / Audio Packet Loss / Audio + // Jitter on the Application Path Details page — the API names + // probably follow the same PlainTitle convention we just learned. + 'AudioMOSScore', + 'AudioMosScore', + 'AudioMOS', + 'AudioMos', + 'AudioPacketLoss', + 'AudioPktLoss', + 'AudioLoss', + 'AudioJitter', + 'AudioBandwidth', + 'AudioBandwidthUsage', + 'ApplicationAudioMOS', + 'ApplicationAudioPacketLoss', + 'ApplicationAudioJitter', + 'MOSScore', + 'VoiceMOS', + 'VoiceMOSScore', + // ─── VIDEO EQUIVALENTS (nice-to-have) ───────────────────────── + 'VideoMOSScore', + 'VideoPacketLoss', + 'VideoJitter', + 'VideoBandwidth', + ]; + + // Metric sweep — uses the HAR-confirmed request shape: + // start_time + end_time (both required), + // view: { individual: "app", summary: false }, + // NO filter.site (per-app queries are tenant-wide-per-app), + // NO filter.app (we're testing WHICH metric names exist for + // any app; adding an app id would narrow to a specific one). + // + // Sweep {gauge, count, Mbps} — the three units we've actually + // observed working. Skip the no-unit case (already known to + // return SCHEMA_CHECK_FAIL: "unit: is missing but it is required"). + const endTime = new Date().toISOString(); + const startTime = new Date(Date.now() - 60 * 60 * 1000).toISOString(); + const metricSweepCandidates = []; + for (const name of perAppMetricNames) { + for (const unitVariant of ['gauge', 'count', 'Mbps']) { + metricSweepCandidates.push({ + method: 'POST', + url: '/sdwan/monitor/v2.6/api/monitor/metrics', + label: `v2.6 monitor/metrics name="${name}", unit="${unitVariant}"`, + body: { + start_time: startTime, + end_time: endTime, + interval: pickInterval5min(), + metrics: [{ name, statistics: ['average'], unit: unitVariant }], + view: { individual: 'app', summary: false }, + }, + }); + } + } + + const combined = []; + for (const c of [...appdefEndpoints, ...metricSweepCandidates]) { + const results = await runCandidates( + c.method || 'POST', c.url, + [{ label: c.label, body: c.body || null }], + {}, args.flags, + ); + combined.push(...results); + } + printSummary('app-list', combined); + + info('If nothing passed, the fastest path from here is:'); + info(' 1. Open the "Application Path Details" page for a site in the Prisma UI'); + info(' 2. Open browser dev tools → Network tab → filter for "monitor" or "metrics"'); + info(' 3. Right-click the request → Copy → Copy as cURL (or copy the request body)'); + info(' 4. Paste the body into: `npm run prisma:probe -- raw POST --body \'\'`'); + info('That surfaces the exact metric name + shape the UI uses.'); + return; + } + + throw new Error( + `unknown try-shapes target "${which}" — expected "health", "lqm", ` + + `"lqm-latency", "lqm-jitter", "lqm-loss", "lqm-mos", ` + + `"app-list", "app-audio-mos", "app-audio-loss", "app-audio-jitter", or "app-audio-bandwidth"`, + ); } /** @@ -741,12 +1101,18 @@ function printHelp() { ' lqm [--metric X] Fetch LQM metric (latency|jitter|loss|mos)', ' alarms [--window minutes] Fetch alarms', ' raw [--body JSON] Arbitrary authenticated request', + ' appdefs [] [--limit N] Dump the tenant app catalog (optional substring filter)', ' try-shapes health Test N healthscore body shapes', ' try-shapes lqm Test N LQM body shapes', ' try-shapes lqm-latency Sweep {name × unit} combos for latency', ' try-shapes lqm-jitter Sweep {name × unit} combos for jitter', ' try-shapes lqm-loss Sweep {name × unit} combos for packet loss', ' try-shapes lqm-mos Sweep {name × unit} combos for MOS', + ' try-shapes app-list Discover per-site app names Prisma sees', + ' try-shapes app-audio-mos [app=rtp-base] Sweep per-app audio MOS shapes', + ' try-shapes app-audio-loss [app=rtp-base] Sweep per-app audio loss shapes', + ' try-shapes app-audio-jitter [app=rtp-base] Sweep per-app audio jitter shapes', + ' try-shapes app-audio-bandwidth [app=rtp-base] Sweep per-app audio BW shapes', '', 'Global flags:', ' --json JSON output', @@ -759,6 +1125,11 @@ function printHelp() { ' node scripts/prismaProbe.js try-shapes health 16158173173100144', ' node scripts/prismaProbe.js lqm 16158173173100144 16158173176610209 --metric loss', ' node scripts/prismaProbe.js try-shapes lqm-loss 16158173173100144 16158173176610209,1666974885552003096', + ' node scripts/prismaProbe.js appdefs rtp', + ' node scripts/prismaProbe.js appdefs --limit 1000', + ' node scripts/prismaProbe.js try-shapes app-list 16158173173100144', + ' node scripts/prismaProbe.js try-shapes app-audio-mos 16158173173100144 15932000365560116', + ' node scripts/prismaProbe.js try-shapes app-audio-loss 16158173173100144 15932000365560116', ' node scripts/prismaProbe.js raw POST /sdwan/v3.7/api/events/query --body \'{"limit":{"count":5}}\'', ].join('\n')); } @@ -781,6 +1152,7 @@ async function main() { lqm: cmdLqm, alarms: cmdAlarms, raw: cmdRaw, + appdefs: cmdAppdefs, 'try-shapes': cmdTryShapes, }; const fn = dispatch[sub]; diff --git a/services/enrichment/sdwanEnrichment.js b/services/enrichment/sdwanEnrichment.js index 952b09b..a3d637d 100644 --- a/services/enrichment/sdwanEnrichment.js +++ b/services/enrichment/sdwanEnrichment.js @@ -60,8 +60,10 @@ import { import { getHealthscore, getLqmMetric, + getAppMetric, getAlarms, } from '../../integrations/paloalto/metrics.js'; +import { buildAppDetailsUrl } from '../../integrations/paloalto/urls.js'; /** * Compose everything /voicediag + /phonestatus care about for a @@ -72,7 +74,10 @@ import { * @param {object} [opts] * @param {number} [opts.windowMinutes] Look-back window for healthscore + LQM. * Also used as the alarm window unless `alarmWindowMinutes` is set. - * Defaults to `WAN_STANDARD_WINDOW_MINUTES` env var or 15. + * Defaults to `WAN_STANDARD_WINDOW_MINUTES` env var or 10080 (7d). + * Widened from 24h because per-app DPI metrics only get datapoints + * when calls actually happen — sporadic stores need a wider window + * for worst-window stats to be meaningful. * @param {number} [opts.alarmWindowMinutes] Override the alarm window * independently (defaults to `max(60, windowMinutes)` since alarms * at < 1h are usually too noisy to be actionable). @@ -104,6 +109,7 @@ export async function collectSdwanForStore(storeNum, opts = {}) { healthscore: null, links: emptyLinks, alarms: emptyAlarms(), + appAudio: null, errors, fetchedAt: new Date().toISOString(), window: { minutes: windowMinutes, alarmMinutes: alarmWindowMinutes }, @@ -129,7 +135,15 @@ export async function collectSdwanForStore(storeNum, opts = {}) { // getHealthscore → site only (filter shape is empty; site match done locally) // getLqmMetric → requires waninterfaceIds (per-path) // getAlarms → site only + // getAppMetric → site + numeric app id. Feature-gated: only + // fires if the voice-app id env var is set. + // Fetches the four audio metrics in parallel, + // gracefully skips them if not configured. const canFetchLqm = waninterfaceIds.length > 0; + const voiceAppCfg = resolveVoiceAppConfig(); + const voiceAppId = voiceAppCfg.appId; + const voiceAppName = voiceAppCfg.appName; + const canFetchAppMetrics = Boolean(voiceAppId); const [ healthscoreRes, latencyRes, @@ -137,6 +151,10 @@ export async function collectSdwanForStore(storeNum, opts = {}) { lossRes, mosRes, alarmsRes, + appMosRes, + appLossRes, + appJitterRes, + appBwRes, ] = await Promise.allSettled([ getHealthscore(site.id, windowMinutes), canFetchLqm ? getLqmMetric(site.id, waninterfaceIds, 'latency', windowMinutes) : Promise.resolve(null), @@ -144,6 +162,10 @@ export async function collectSdwanForStore(storeNum, opts = {}) { canFetchLqm ? getLqmMetric(site.id, waninterfaceIds, 'loss', windowMinutes) : Promise.resolve(null), canFetchLqm ? getLqmMetric(site.id, waninterfaceIds, 'mos', windowMinutes) : Promise.resolve(null), getAlarms(site.id, alarmWindowMinutes), + canFetchAppMetrics ? getAppMetric(site.id, voiceAppId, 'mos', windowMinutes) : Promise.resolve(null), + canFetchAppMetrics ? getAppMetric(site.id, voiceAppId, 'loss', windowMinutes) : Promise.resolve(null), + canFetchAppMetrics ? getAppMetric(site.id, voiceAppId, 'jitter', windowMinutes) : Promise.resolve(null), + canFetchAppMetrics ? getAppMetric(site.id, voiceAppId, 'bandwidth', windowMinutes) : Promise.resolve(null), ]); recordFailure(errors, 'healthscore', healthscoreRes); @@ -152,6 +174,16 @@ export async function collectSdwanForStore(storeNum, opts = {}) { recordFailure(errors, 'lqm.loss', lossRes); recordFailure(errors, 'lqm.mos', mosRes); recordFailure(errors, 'alarms', alarmsRes); + if (canFetchAppMetrics) { + // Scope names stay stable across app changes so the check + // envelope (getAppAudio) doesn't need to know which app is + // configured — the semantics ("audio MOS for the voice app") + // are what matter, not which RFC 3550 app it is. + recordFailure(errors, 'app.voice.mos', appMosRes); + recordFailure(errors, 'app.voice.loss', appLossRes); + recordFailure(errors, 'app.voice.jitter', appJitterRes); + recordFailure(errors, 'app.voice.bandwidth', appBwRes); + } // Healthscore response covers the whole tenant (see filter shape // note in metrics.js). Match the requested site locally. @@ -206,15 +238,39 @@ export async function collectSdwanForStore(storeNum, opts = {}) { } } + // Parse app-metric time series into {avg, min, max, samples} tuples + // per metric. Only present when the voice-app env is configured + // (PRISMA_APP_ID_VOICE, with PRISMA_APP_ID_RTP_BASE honored for + // backwards compatibility). + // + // `detailsUrl` is the SCM UI deep-link into "Application Path + // Details" for this exact (site, app) pair — computed here so both + // renderers (WAN diag + voice diag) share the same URL without + // duplicating the pattern. + const appAudio = canFetchAppMetrics + ? { + appId: voiceAppId, + appName: voiceAppName, + detailsUrl: buildAppDetailsUrl(site.id, voiceAppId), + mos: summarizeAppSeries(valueOf(appMosRes), 'AppAudioMos'), + loss: summarizeAppSeries(valueOf(appLossRes), 'AppPerfUDPAudioPacketLoss'), + jitter: summarizeAppSeries(valueOf(appJitterRes), 'AppPerfUDPAudioJitter'), + bandwidth: summarizeAppSeries(valueOf(appBwRes), 'AppPerfUDPAudioBandwidth'), + } + : null; + const elapsed = Date.now() - startedAt; const alarmDiag = alarms?._diag ? ` alarms=${alarms._diag.rawEventCount}raw→${alarms._diag.rawEventCount - alarms._diag.droppedForSiteMismatch}scoped` : ''; + const appAudioDiag = appAudio + ? ` app=${appAudio.appName}(${appAudio.appId}) app.mos=[${appAudio.mos?.samples || 0}pts,min=${appAudio.mos?.min ?? 'n/a'}]` + : ''; logger( 'sdwan:enrich', `store ${storeNum} → site ${site.name} (${site.id}): ${elements.length} elements, ` + - `${links.length} links, health=${healthscore?.value ?? 'n/a'},${alarmDiag} errors=${errors.length}, ` + - `window=${windowMinutes}m/alarms=${alarmWindowMinutes}m, ${elapsed}ms`, + `${links.length} links, health=${healthscore?.value ?? 'n/a'},${alarmDiag}${appAudioDiag} ` + + `errors=${errors.length}, window=${windowMinutes}m/alarms=${alarmWindowMinutes}m, ${elapsed}ms`, ); return { @@ -235,6 +291,10 @@ export async function collectSdwanForStore(storeNum, opts = {}) { healthscore, links, alarms, + // Per-application "Application Path Details" style metrics — real + // voice-quality signal from DPI on actual RTP traffic. Null when + // PRISMA_APP_ID_RTP_BASE is not configured (feature-gated). + appAudio, errors, fetchedAt: new Date().toISOString(), // Report the effective look-back so renderers can surface it and @@ -243,27 +303,166 @@ export async function collectSdwanForStore(storeNum, opts = {}) { }; } -// Default WAN look-back window (24h in minutes). Chosen because the -// primary users of /phonestatus + /voicediag are triaging trouble -// tickets after the fact, where "was voice quality bad today?" is -// more useful than "how does it look right this second?". Operators -// wanting a live snapshot can pass `--window 15m` on /voicediag or -// set WAN_STANDARD_WINDOW_MINUTES=15 in the environment. -const DEFAULT_WINDOW_MINUTES = 1440; +// ─── App-metric time-series aggregator ────────────────────────────── +// +// App metrics come back as a real time series (usually 24h × 5min = +// 288 datapoints) rather than a single aggregated number like LQM. +// For voice quality, the worst-window matters more than the average: +// a store that averages 4.0 MOS over 24h but dipped to 1.85 for +// several 5-min windows had unusable calls in those windows even +// though the average looks fine. +// +// This helper extracts {avg, min, max, p95, samples} from the +// standard v2.6 response shape: +// metrics[0].series[0].data[0].datapoints[{time, value}] +// +// Returns null for missing / malformed responses. All-null datapoints +// (e.g. TCPFlowCount for a UDP-only app in the HAR) return +// { samples: N, avg/min/max: null } so downstream callers can +// distinguish "we asked but Prisma had no data" from "we didn't ask". + +/** + * @param {object} raw v2.6 monitor/metrics response body + * @param {string} expectedName metric name we expect at metrics[].series[].name + * @returns {{avg, min, max, p95, samples, expectedName}|null} + */ +export function summarizeAppSeries(raw, expectedName) { + if (!raw || typeof raw !== 'object') return null; + const series = raw.metrics?.[0]?.series?.[0]; + if (!series) return null; + // Sanity check: match by name so a mis-wired call can't silently + // land a jitter response into the loss bucket. + if (expectedName && series.name && series.name !== expectedName) { + logger('sdwan:enrich', + `summarizeAppSeries: expected metric name "${expectedName}", got "${series.name}"`, 'warn'); + } + const datapoints = series.data?.[0]?.datapoints; + if (!Array.isArray(datapoints)) return null; + + const values = datapoints + .map((p) => (typeof p?.value === 'number' && Number.isFinite(p.value) ? p.value : null)) + .filter((v) => v !== null); + + const summary = { + expectedName, + unit: series.unit || null, + interval: series.interval || null, + samples: datapoints.length, + validSamples: values.length, + avg: null, min: null, max: null, p95: null, + // Keep the raw values around so a check can compute additional + // stats (percent-of-samples-above-threshold, etc.) without having + // to re-parse the raw response. Small (<= 288 numbers per metric) + // so no memory concern. + values, + }; + + if (values.length === 0) return summary; + + let sum = 0, min = values[0], max = values[0]; + for (const v of values) { sum += v; if (v < min) min = v; if (v > max) max = v; } + const sorted = [...values].sort((a, b) => a - b); + const p95Idx = Math.min(sorted.length - 1, Math.floor(0.95 * sorted.length)); + + summary.avg = round2(sum / values.length); + summary.min = round2(min); + summary.max = round2(max); + summary.p95 = round2(sorted[p95Idx]); + return summary; +} + +// ─── Voice-application (DPI) env resolution ───────────────────────── +// +// The per-app DPI feature is generic — different tenants may target +// different voice apps (rtp-base, Webex_Calling_RTP, MS Teams RTP, +// Zoom, etc.). Rather than hardcode a specific app, resolve id + +// display name from env at request time. +// +// Precedence: +// 1. PRISMA_APP_ID_VOICE — canonical, generic env var +// 2. PRISMA_APP_ID_RTP_BASE — backwards-compat with the +// original release, logged as +// deprecated on use +// +// Display name defaults to the app-key hint ('rtp-base' if only the +// legacy env is set, 'voice' otherwise); PRISMA_APP_NAME_VOICE +// overrides for the operator's preferred UI label. +// +// Exported so tests can exercise the resolution logic without +// spinning up the full enrichment path. +let _rtpBaseDeprecationLogged = false; +export function resolveVoiceAppConfig() { + const genericId = process.env.PRISMA_APP_ID_VOICE; + const legacyId = process.env.PRISMA_APP_ID_RTP_BASE; + const nameOverride = process.env.PRISMA_APP_NAME_VOICE; + + let appId = null; + let defaultName = 'voice'; + + if (genericId && genericId.trim()) { + appId = genericId.trim(); + } else if (legacyId && legacyId.trim()) { + appId = legacyId.trim(); + defaultName = 'rtp-base'; + if (!_rtpBaseDeprecationLogged) { + logger('sdwan:enrich', + 'PRISMA_APP_ID_RTP_BASE is deprecated — rename to PRISMA_APP_ID_VOICE ' + + '(and optionally set PRISMA_APP_NAME_VOICE for a friendly display label). ' + + 'The old name will continue to work but will be removed in a future release.', + 'warn'); + _rtpBaseDeprecationLogged = true; + } + } + + const appName = (nameOverride && nameOverride.trim()) + ? nameOverride.trim() + : defaultName; + + return { appId, appName }; +} + +/** @internal Test-only: reset the once-per-process deprecation flag. */ +export function _resetVoiceAppDeprecationFlag() { + _rtpBaseDeprecationLogged = false; +} + +function round2(n) { + if (!Number.isFinite(n)) return null; + return Math.round(n * 100) / 100; +} + +// Default WAN look-back window (7 days in minutes). Widened from 24h +// because per-app DPI metrics (Webex_Calling_RTP etc.) only get +// datapoints WHEN CALLS HAPPEN — a store that only takes 3-4 Webex +// calls per day gives us ~30 samples in 24h, well below the point +// where "worst-window" stats mean anything. Seven days consistently +// yields 150-300+ per-app samples across stores of any size and +// still keeps the LQM/healthscore signal meaningful. The trade-off +// is that a fresh outage is diluted in a 7-day average, so: +// - The app-metric checks grade against WORST-window (already do), +// not average, so this doesn't blunt their signal. +// - Operators triaging a live incident should pass `--window 1h` +// or `--window 24h` (or set WAN_STANDARD_WINDOW_MINUTES) to +// narrow the view. The env override still works exactly the same. +const DEFAULT_WINDOW_MINUTES = 10080; // 7 × 24 × 60 +const MAX_WINDOW_MINUTES = 10080; // hard cap — see pickInterval /** * Snap an arbitrary window request to a reasonable, sane value. - * Falls back to the env default (WAN_STANDARD_WINDOW_MINUTES) or 1440 - * (24h). Prisma's own upper bound seems to sit around 30 days for - * aiops queries, but voice-quality signal degrades past a day of - * averaging so we cap at 24h for now. + * Falls back to the env default (WAN_STANDARD_WINDOW_MINUTES) or + * 10080 (7d). Prisma's own upper bound sits at ~30 days for aiops + * queries, but per-app DPI series past 7d hit interval-collapse + * (Prisma downsamples to '1day', which loses the transient windows + * these checks are designed to catch). We hard-cap at 7d to keep + * the guarantees consistent — set MAX_WINDOW_MINUTES higher if you + * ever need to push past it, and re-verify pickAppMetricInterval / + * pickInterval interact cleanly with the new upper bound. */ function normalizeWindowMinutes(m) { const raw = Number.isFinite(Number(m)) && Number(m) > 0 ? Number(m) : Number(process.env.WAN_STANDARD_WINDOW_MINUTES) || DEFAULT_WINDOW_MINUTES; - // Clamp to [1, 1440] (1 min .. 24 h). - return Math.max(1, Math.min(1440, Math.floor(raw))); + return Math.max(1, Math.min(MAX_WINDOW_MINUTES, Math.floor(raw))); } // ─── Response normalisers ───────────────────────────────────────── diff --git a/services/renderers/voiceDiagRenderer.js b/services/renderers/voiceDiagRenderer.js index e7102be..1a89e71 100644 --- a/services/renderers/voiceDiagRenderer.js +++ b/services/renderers/voiceDiagRenderer.js @@ -181,6 +181,13 @@ function renderDetails(details) { if ('siteId' in details && 'siteName' in details) { return renderSiteDetails(details); } + // Per-app audio MOS/loss/jitter summary — has + // {avg,min,max,worst,badSampleCount,badSamplePct,appName,samples, + // validSamples,interval,warnThresh,errorThresh}. Match on the + // combination unique to that shape. + if ('appName' in details && 'worst' in details && 'validSamples' in details) { + return renderAppAudioDetails(details); + } if ('value' in details && 'warnThresh' in details && 'errorThresh' in details) { return renderThresholdDetails(details); } @@ -247,6 +254,66 @@ function renderLinkStateDetails(details) { return lines.join('\n'); } +/** + * Per-app audio-quality (MOS/loss/jitter) detail formatter. + * The summary carries: {appName, worst, avg, min, max, p95, + * samples, validSamples, interval, warnThresh, errorThresh, + * badSampleCount, badSamplePct, standardLabel, unit}. + * + * Output (example, MOS): + * + * - App: Webex_Calling_RTP (voice traffic, DPI) + * - Threshold: warn < 4, error < 3.5 + * - Worst window: 1.85 · Avg: 4.03 · p95: 4.28 + * - Range: 1.85 – 4.41 across 288/288 samples @ 5min + * - Time in warn/error: 42 samples (15%) + */ +function renderAppAudioDetails(details) { + const { + appName, worst, avg, min, max, p95, + samples, validSamples, interval, + warnThresh, errorThresh, standardLabel, unit, + badSampleCount, badSamplePct, + detailsUrl, + } = details; + + const u = unit || ''; + const lines = [` - App: ${appName || 'voice'} (voice traffic, DPI)`]; + + if (standardLabel) { + lines.push(` - Threshold: ${standardLabel}`); + } else if (Number.isFinite(warnThresh) || Number.isFinite(errorThresh)) { + lines.push(` - Threshold: warn @ ${warnThresh ?? '?'}, error @ ${errorThresh ?? '?'}`); + } + + const worstStr = worst == null ? '—' : `${worst}${u}`; + const avgStr = avg == null ? '—' : `${avg}${u}`; + const p95Str = p95 == null ? '—' : `${p95}${u}`; + lines.push(` - Worst window: **${worstStr}** · Avg: ${avgStr} · p95: ${p95Str}`); + + if (min != null && max != null) { + lines.push(` - Range: ${min}${u} – ${max}${u} across ${validSamples}/${samples} samples @ ${interval || '5min'}`); + } + + if (Number.isFinite(badSampleCount) && Number.isFinite(badSamplePct)) { + if (badSampleCount > 0) { + lines.push(` - Time in warn/error: ${badSampleCount} sample${badSampleCount === 1 ? '' : 's'} (~${badSamplePct}% of window)`); + } else { + lines.push(` - Time in warn/error: 0 samples (clean throughout window)`); + } + } + + // SCM UI deep-link — always the last row so it acts as a "next + // step" for anyone inspecting the details. Only rendered when the + // enrichment layer was able to build a URL (needs both site.id + + // appAudio.appId — either missing → null skips this line cleanly). + if (detailsUrl) { + lines.push(` - [View in Prisma UI](${detailsUrl})`); + } + + return lines.join('\n'); +} + /** * Alarm counts + optional category breakdown + rolled-up recent * samples. Handles both the old shape (bare `critical/major/minor + diff --git a/services/renderers/wanDiagnosticsRenderer.js b/services/renderers/wanDiagnosticsRenderer.js index 217af1f..3508eeb 100644 --- a/services/renderers/wanDiagnosticsRenderer.js +++ b/services/renderers/wanDiagnosticsRenderer.js @@ -92,6 +92,20 @@ export function renderWanDiagnosticsMarkdown(data, opts = {}) { out += `_No WAN path metrics available for this site._\n`; } + // Per-app "Application Path Details" section — real voice-quality + // signal from DPI on actual RTP traffic. Which app is measured + // depends on the tenant's configured voice app (Webex_Calling_RTP, + // rtp-base, MS_Teams_RTP, etc.). Shown BEFORE alarms because when + // it fires, it's usually more actionable than an overlay alarm: + // it's "your calls sounded bad at these times", not "a tunnel + // bounced but recovered". Feature-gated on PRISMA_APP_ID_VOICE + // (backwards-compat: PRISMA_APP_ID_RTP_BASE) — omitted entirely + // when not configured so this doesn't add empty sections for + // tenants that opted out. + if (data.appAudio) { + out += renderAppAudioSection(data.appAudio, data.window); + } + // Alarms summary (only if any are active). Samples are rolled up // by (code + severity) so 20 identical NETWORK_ANYNETLINK_DOWN // events show as one line with "×20" instead of pasting 20 nearly- @@ -137,7 +151,10 @@ export function renderWanDiagnosticsMarkdown(data, opts = {}) { } if (footer) { - out += `\n*WAN metrics pulled at ${new Date().toLocaleTimeString()} from Prisma SD-WAN. Use \`/voicediag ${storeNum || data.storeNum} --only wanLatency,wanJitter,wanLoss,wanMos,wanHealthscore,wanLinkState,wanAlarms\` for pass/warn/error breakdowns.*`; + const store = storeNum || data.storeNum; + out += `\n*WAN metrics pulled at ${new Date().toLocaleTimeString()} from Prisma SD-WAN. ` + + `Use \`/voicediag ${store} --only wanLatency,wanJitter,wanLoss,wanMos,wanHealthscore,wanLinkState,wanAlarms\` for link-probe breakdowns, ` + + `or \`/voicediag ${store} --only wanAppRtpMos,wanAppRtpLoss,wanAppRtpJitter\` for per-app RTP quality.*`; } return out.trim(); @@ -157,6 +174,85 @@ function humanWindowLabel(minutes) { return `${minutes}m`; } +/** + * Render the "Voice Traffic Quality" section — real DPI measurements + * on actual RTP frames rather than synthetic link probes. Shows + * worst-window / avg / % of samples degraded per metric, with icons + * keyed off the WORST-window value (that's the signal that actually + * correlates with "the operator got a bad-calls ticket"). + * + * Section header includes the configured voice-app name (e.g. + * "Webex_Calling_RTP", "rtp-base") so the operator knows which DPI + * signature was measured — different apps have very different + * traffic patterns and one may show issues the other misses. + * + * @param {object} appAudio ctx.sdwanData.appAudio + * @param {object} window ctx.sdwanData.window + */ +function renderAppAudioSection(appAudio, window) { + if (!appAudio) return ''; + const { mos, loss, jitter, bandwidth, appName, detailsUrl } = appAudio; + // If literally every metric is null/empty, skip entirely — usually + // means the tenant has the env var set but this site has no RTP + // traffic yet. + const anyData = [mos, loss, jitter, bandwidth].some( + (s) => s && s.validSamples > 0, + ); + if (!anyData) return ''; + + const winLabel = window?.minutes ? humanWindowLabel(window.minutes) : '7d'; + // Deep-link into the Strata Cloud Manager "Application Path + // Details" page for this site+app. Only rendered when a URL was + // computed (detailsUrl is null if either id is missing). + const linkSuffix = detailsUrl + ? ` — [View in Prisma UI](${detailsUrl})` + : ''; + + let out = `\n**Voice Traffic Quality (${appName || 'voice'}, last ${winLabel})**${linkSuffix}\n`; + out += `_Measured on real RTP frames via Prisma DPI — worst-window matters more than avg for voice._\n`; + + const mosLine = fmtAppMetricLine('MOS', mos, '', 3.5, 4.0, /* lowIsBad */ true); + const lossLine = fmtAppMetricLine('Loss', loss, '%', 15, 5, /* lowIsBad */ false); + const jitterLine = fmtAppMetricLine('Jitter', jitter, 'ms', 50, 30, /* lowIsBad */ false); + const bwLine = fmtAppMetricLine('Bandwidth', bandwidth, 'Mbps', null, null, /* lowIsBad */ false); + + for (const line of [mosLine, lossLine, jitterLine, bwLine].filter(Boolean)) { + out += `- ${line}\n`; + } + + return out; +} + +/** + * One row of the voice-traffic-quality section. + * + * @param {string} label + * @param {object|null} summary {avg, min, max, samples, validSamples, ...} + * @param {string} unit display unit ("%", "ms", "", "Mbps") + * @param {number|null} errThresh + * @param {number|null} warnThresh + * @param {boolean} lowIsBad true for MOS (lower is worse) + */ +function fmtAppMetricLine(label, summary, unit, errThresh, warnThresh, lowIsBad) { + if (!summary) return ''; + if (summary.validSamples === 0) { + return `❓ **${label}** — no data (${summary.samples || 0} samples, all null)`; + } + const worst = lowIsBad ? summary.min : summary.max; + let icon; + if (errThresh == null && warnThresh == null) { + icon = 'ℹ️'; // no threshold → context-only (bandwidth) + } else { + icon = iconFromNumeric(worst, errThresh, warnThresh, lowIsBad); + } + const worstLabel = lowIsBad ? 'worst (lowest)' : 'worst'; + return ( + `${icon} **${label}** — ${worstLabel}: **${worst}${unit}** • ` + + `avg: ${summary.avg}${unit} • ${summary.validSamples}/${summary.samples} samples ` + + `@ ${summary.interval || '5min'}` + ); +} + function renderOneLink(link, t) { const nameLabel = link.interfaceName || link.interfaceId; const transport = link.transportType ? ` [${link.transportType}]` : ''; diff --git a/services/voiceDiag/README.md b/services/voiceDiag/README.md index 2e7693b..f766795 100644 --- a/services/voiceDiag/README.md +++ b/services/voiceDiag/README.md @@ -12,19 +12,24 @@ objects, and hands them to the renderer + adaptive-card layer in /voicediag default: hides OK checks, posts fixable cards /voicediag detail include OK checks + expand every details block /voicediag --only dnd,callForwarding -/voicediag --window 15m narrow the WAN look-back (default 24h) +/voicediag --window 24h narrow the WAN look-back (default 7d) /voicediag list-checks enumerate every registered check + its scope ``` HTTP path: `GET /voicediag?storeNum=[&detailed=true][&only=dnd,callWaiting][&window=15m]`. `--window` accepts `Nm` / `Nh` / `Nd` shorthand (e.g. `15m`, `1h`, `6h`, -`24h`, `1d`) or a bare integer of minutes. Applies to the WAN -healthscore + LQM (per-path latency/jitter/loss/MOS) fetches. Alarms -are floored at 60m regardless — sub-hour alarm queries are usually -too noisy to be actionable. Global default is -`WAN_STANDARD_WINDOW_MINUTES` (env, defaults to 1440 / 24h). The -same default is used by the `/phonestatus` WAN follow-up (no CLI +`24h`, `1d`, `7d`) or a bare integer of minutes; hard-capped at 7d. +Applies to the WAN healthscore + LQM (per-path latency/jitter/loss/MOS) ++ per-app voice-DPI fetches + alarms. Alarms are floored at 60m +regardless — sub-hour alarm queries are usually too noisy to be +actionable. Global default is `WAN_STANDARD_WINDOW_MINUTES` (env, +defaults to `10080` / 7 days). The 7d default was chosen because +per-app DPI metrics (Webex_Calling_RTP etc.) only get datapoints when +calls actually happen — sporadic stores need a wider window for +worst-window statistics to be meaningful. For live-incident triage +where you want a fresh snapshot, pass `--window 1h` or `--window 24h`. +The same default is used by the `/phonestatus` WAN follow-up (no CLI override on that surface — set the env if you want a different value globally). HTTP callers get the markdown snapshot only — remediation cards are @@ -147,6 +152,9 @@ in parallel, and normalises to a stable shape the checks + the | `wanJitter` | <= `WAN_STANDARD_JITTER_WARN_MS` (default 30ms) | warn > 30ms, **error** > 50ms | RFC 3550 jitter-buffer envelope | | `wanLoss` | <= `WAN_STANDARD_LOSS_WARN_PCT` (default 1%) | warn > 1%, **error** > 3% | G.711 PLC tolerance | | `wanMos` | >= `WAN_STANDARD_MOS_WARN` (default 4.0) | warn < 4.0, **error** < 3.5 | ITU-T P.800 MOS scale. LOW is bad — the accessor flips the comparison. | +| `wanAppRtpMos` | Worst 5-min window >= `WAN_STANDARD_APP_MOS_WARN` (default 4.0) | warn < 4.0, **error** < 3.5 | Real DPI measurement on the configured voice app's RTP frames (e.g. `Webex_Calling_RTP`, `rtp-base`). Graded against the WORST window in the series — catches transient degradation link-probe averages hide. Skipped when `PRISMA_APP_ID_VOICE` unset. | +| `wanAppRtpLoss` | Worst 5-min window <= `WAN_STANDARD_APP_LOSS_WARN_PCT` (default 5%) | warn > 5%, **error** > 15% | DPI packet-loss for the same voice app. Same feature gate. | +| `wanAppRtpJitter` | Worst 5-min window <= `WAN_STANDARD_APP_JITTER_WARN_MS` (default 30ms) | warn > 30ms, **error** > 50ms | DPI jitter for the same voice app. Same feature gate. | | `wanAlarms` | Zero critical + zero major alarms in the last hour | warn on major, **error** on critical | Minor alarms are info-only. Pass-through of Prisma severity. | Environment overrides (all optional — defaults match the standards @@ -164,7 +172,16 @@ above): | `WAN_STANDARD_MOS_ERROR` | `3.5` | MOS error threshold | | `WAN_STANDARD_HEALTHSCORE_WARN` | `80` | Site healthscore warn threshold (0-100) | | `WAN_STANDARD_HEALTHSCORE_ERROR` | `60` | Site healthscore error threshold | -| `WAN_STANDARD_ENABLED` | `true` | Global kill-switch for the WAN bucket. `false` silences all 8 WAN checks while a Prisma cleanup / integration validation is in flight. | +| `WAN_STANDARD_APP_MOS_WARN` | `4.0` | Per-app worst-window MOS warn threshold | +| `WAN_STANDARD_APP_MOS_ERROR` | `3.5` | Per-app worst-window MOS error threshold | +| `WAN_STANDARD_APP_LOSS_WARN_PCT` | `5` | Per-app worst-window packet-loss warn threshold, % | +| `WAN_STANDARD_APP_LOSS_ERROR_PCT` | `15` | Per-app worst-window packet-loss error threshold, % | +| `WAN_STANDARD_APP_JITTER_WARN_MS` | `30` | Per-app worst-window jitter warn threshold, ms | +| `WAN_STANDARD_APP_JITTER_ERROR_MS` | `50` | Per-app worst-window jitter error threshold, ms | +| `PRISMA_APP_ID_VOICE` | _(unset)_ | Numeric Prisma app id for the tenant's voice app. Recommended: `Webex_Calling_RTP` for Webex Calling shops (excludes non-Webex UDP noise), otherwise `rtp-base` for the generic RTP signature. When set, /phonestatus + /voicediag pull real DPI voice-quality metrics from Prisma's Application Path Details endpoint. Discover with `npm run prisma:probe -- appdefs webex` (or `rtp`) or from the Prisma UI URL. Leave unset to skip the extra API calls. | +| `PRISMA_APP_NAME_VOICE` | `voice` | Display label for the app configured above. Set to the exact Prisma UI label (e.g. `Webex_Calling_RTP`) so section headers and check messages cross-reference cleanly. | +| `PRISMA_APP_ID_RTP_BASE` | _(unset)_ | **Deprecated.** Backwards-compat fallback for the original release. Still honored if set and `PRISMA_APP_ID_VOICE` is unset (with a one-time deprecation warning in the log); display name defaults to `rtp-base` in that case. Rename to `PRISMA_APP_ID_VOICE` at your convenience. | +| `WAN_STANDARD_ENABLED` | `true` | Global kill-switch for the WAN bucket. `false` silences all 11 WAN checks while a Prisma cleanup / integration validation is in flight. | Prisma credentials and auth-mode selection live under the `Palo Alto Prisma SD-WAN` block in `.env.example` (SASE OAuth 2.0 diff --git a/services/voiceDiag/checks/index.js b/services/voiceDiag/checks/index.js index 63472f6..a9241be 100644 --- a/services/voiceDiag/checks/index.js +++ b/services/voiceDiag/checks/index.js @@ -50,6 +50,9 @@ import { wanLatencyCheck } from './wan/wanLatency.js'; import { wanJitterCheck } from './wan/wanJitter.js'; import { wanLossCheck } from './wan/wanLoss.js'; import { wanMosCheck } from './wan/wanMos.js'; +import { wanAppRtpMosCheck } from './wan/wanAppRtpMos.js'; +import { wanAppRtpLossCheck } from './wan/wanAppRtpLoss.js'; +import { wanAppRtpJitterCheck } from './wan/wanAppRtpJitter.js'; import { wanAlarmsCheck } from './wan/wanAlarms.js'; // Order: user-facing feature signals first (things an operator can @@ -77,6 +80,15 @@ export const CHECKS = [ wanJitterCheck, wanLossCheck, wanMosCheck, + // Per-app (DPI) audio-quality checks. These are graded against the + // WORST 5-min window in the series rather than the average, so they + // catch transient degradation the LQM per-link averages smooth over. + // Feature-gated on the PRISMA_APP_ID_VOICE env var (with + // PRISMA_APP_ID_RTP_BASE honored for backwards compat) — skipped + // with an actionable message when not configured. + wanAppRtpMosCheck, + wanAppRtpLossCheck, + wanAppRtpJitterCheck, wanAlarmsCheck, phoneOnlineCheck, ]; diff --git a/services/voiceDiag/checks/wan/_helpers.js b/services/voiceDiag/checks/wan/_helpers.js index 01405b8..f659964 100644 --- a/services/voiceDiag/checks/wan/_helpers.js +++ b/services/voiceDiag/checks/wan/_helpers.js @@ -75,6 +75,28 @@ export const getMosError = () => num('WAN_STANDARD_MOS_ERROR', 3. export const getHealthscoreWarn = () => num('WAN_STANDARD_HEALTHSCORE_WARN', 80); export const getHealthscoreError = () => num('WAN_STANDARD_HEALTHSCORE_ERROR', 60); +// ─── Per-application (DPI) audio-quality thresholds ────────────── +// Applied against the WORST-window value in the series (not the avg) +// since these signals come as 288-point 24h time series and the whole +// point of app-quality checks is to catch transient degradation the +// LQM link-probe average smooths away. Defaults match the ITU-T +// references but are DELIBERATELY less strict than the link-probe +// thresholds because false positives from a single bad 5-min window +// would drown the operator in noise. Tune via env if your traffic +// mix is more sensitive. +// +// MOS (worst-window, lower is worse — call quality drops) +export const getAppMosWarn = () => num('WAN_STANDARD_APP_MOS_WARN', 4.0); +export const getAppMosError = () => num('WAN_STANDARD_APP_MOS_ERROR', 3.5); + +// Loss (worst-window percentage, higher is worse) +export const getAppLossWarnPct = () => num('WAN_STANDARD_APP_LOSS_WARN_PCT', 5); +export const getAppLossErrorPct = () => num('WAN_STANDARD_APP_LOSS_ERROR_PCT', 15); + +// Jitter (worst-window ms, higher is worse) +export const getAppJitterWarnMs = () => num('WAN_STANDARD_APP_JITTER_WARN_MS', 30); +export const getAppJitterErrorMs = () => num('WAN_STANDARD_APP_JITTER_ERROR_MS', 50); + // ─── Utilities used by multiple checks ──────────────────────────── /** @@ -97,6 +119,242 @@ export function labelForLink(link) { return `${name}${el}${tx}`; } +/** + * Accessor for the per-app audio-quality summary computed in + * sdwanEnrichment.js. Returns a small envelope so the caller can + * distinguish four states cleanly: + * + * {configured:false, summary:null, fetchError:null} + * Voice-app env not set → checks return "not configured" + * skipped. See sdwanEnrichment.js:resolveVoiceAppConfig() for + * the env-var contract (PRISMA_APP_ID_VOICE, or + * PRISMA_APP_ID_RTP_BASE for backwards compat). This is a + * config decision, not a transient failure. + * + * {configured:true, summary:null, fetchError:''} + * Env set + Prisma fetch failed (429 storm, timeout, schema + * drift). Different UX from "not configured" — surfaces the + * real error so the operator can retry or fix. + * + * {configured:true, summary:{...},fetchError:null} + * Fetch succeeded but Prisma returned zero valid samples + * (no matching RTP traffic in the window). Skipped with a + * "no traffic seen" message. + * + * {configured:true, summary:{...validSamples>0...}, fetchError:null} + * Green path — check grades against the summary. + * + * @param {object} ctx + * @param {'mos'|'loss'|'jitter'|'bandwidth'} which + */ +export function getAppAudio(ctx, which) { + const container = ctx?.sdwanData?.appAudio || null; + if (!container) { + return { + configured: false, + summary: null, + fetchError: null, + appName: null, + detailsUrl: null, + }; + } + // Error-scope names are app-agnostic ("app.voice.") so the + // check doesn't have to know which specific voice app is + // configured — it just asks "was the fetch for MOS on the voice + // app OK?" See sdwanEnrichment.js:recordFailure() for the writer. + const scope = `app.voice.${which}`; + const errRec = Array.isArray(ctx?.sdwanData?.errors) + ? ctx.sdwanData.errors.find((e) => e?.scope === scope) + : null; + return { + configured: true, + summary: container[which] || null, + fetchError: errRec?.message || null, + appName: container.appName || 'voice', + // SCM UI deep-link computed once in sdwanEnrichment.js. Threaded + // through here so the check details carry it into the voiceDiag + // renderer as a "View in Prisma UI" link. + detailsUrl: container.detailsUrl || null, + }; +} + +/** + * Humanize the raw Prisma unit string into a display-friendly suffix. + * The API returns strings like `"percentage"` and `"milliseconds"` + * which read like error messages when concatenated into a UI value + * (`"11.83percentage"`). This normalizes to the units an operator + * actually recognizes. + * + * Exposed so both the renderer and the check message use the same + * mapping — divergence there would produce mismatched icons vs + * numbers. + */ +export function humanizeMetricUnit(raw) { + if (!raw) return ''; + const lc = String(raw).toLowerCase(); + if (lc === 'percentage' || lc === 'percent') return '%'; + if (lc === 'milliseconds' || lc === 'ms') return 'ms'; + if (lc === 'count' || lc === 'gauge' || lc === 'score') return ''; + if (lc === 'kbps') return 'kbps'; + if (lc === 'mbps') return 'Mbps'; + if (lc === 'bps') return 'bps'; + if (lc === 'seconds' || lc === 's') return 's'; + // Anything else — return as-is, prefixed with a space so it's + // visually separated from the number (`"12 something"` reads + // better than `"12something"`). + return ` ${raw}`; +} + +/** + * Grade a per-app audio quality summary against warn/error thresholds. + * Handles the four cases the envelope from `getAppAudio()` can be in: + * not-configured, fetch-failed, no-traffic, gradeable. + * + * @param {object} args + * @param {object} args.audio envelope from getAppAudio(): + * {configured, summary, fetchError, appName} + * @param {string} args.label display label ("audio MOS", "audio loss", …) + * @param {string} args.unit display unit ("%", "ms", "") + * @param {number} args.warnThresh + * @param {number} args.errorThresh + * @param {boolean} args.lowIsBad true → warn/error when the + * WORST-window VALUE FALLS BELOW + * the threshold (MOS). false → + * warn/error when the WORST-window + * value RISES ABOVE (loss, jitter). + * @param {string} [args.standardLabel] optional inline "warn > X" + * description embedded in the message + * @returns {object} CheckResult + */ +export function evaluateAppAudioMetric({ + audio, label, unit, warnThresh, errorThresh, lowIsBad = false, + standardLabel, +}) { + const appName = audio?.appName || 'the voice app'; + + if (!audio || audio.configured === false) { + return { + status: 'skipped', + message: + `Per-app ${label} unavailable — set PRISMA_APP_ID_VOICE in the ` + + `environment (with optional PRISMA_APP_NAME_VOICE for the display ` + + `name) to enable per-application DPI metrics.`, + details: null, + remediation: null, + }; + } + + if (audio.fetchError) { + return { + status: 'skipped', + message: + `Per-app ${label} for ${appName}: Prisma fetch failed — ${audio.fetchError}. ` + + `Common cause: parallel-request rate limiting (429). Retry in ~30 seconds.`, + details: { fetchError: audio.fetchError, appName }, + remediation: null, + }; + } + + const summary = audio.summary; + if (!summary) { + return { + status: 'skipped', + message: + `Per-app ${label} for ${appName}: no summary produced (Prisma returned an ` + + `empty or malformed response). Check paloalto:metrics logs for the raw payload.`, + details: null, + remediation: null, + }; + } + if (!summary.validSamples || summary.validSamples === 0) { + return { + status: 'skipped', + message: + `Per-app ${label} for ${appName}: Prisma returned ` + + `${summary.samples || 0} datapoint(s), all null — no voice traffic ` + + `matched by DPI in the window.`, + details: summary, + remediation: null, + }; + } + + const worst = lowIsBad ? summary.min : summary.max; + const grade = (v) => { + if (v == null) return 'skipped'; + if (lowIsBad) { + if (v < errorThresh) return 'error'; + if (v < warnThresh) return 'warn'; + return 'ok'; + } + if (v > errorThresh) return 'error'; + if (v > warnThresh) return 'warn'; + return 'ok'; + }; + + const status = grade(worst); + + // Percent of samples that were in warn or error range — the "how + // often was it bad" number that turns the worst-window verdict into + // an actionable "N% of your voice calls saw this" statistic. + const bad = summary.values.filter( + (v) => grade(v) === 'warn' || grade(v) === 'error', + ).length; + const badPct = summary.validSamples > 0 + ? Math.round((bad / summary.validSamples) * 100) + : 0; + + // Store both the raw API unit (for debugging) and the humanized + // display unit (what the renderer + message use). Overriding the + // spread unit with the check-arg unit lets each check pick a nice + // suffix ('%', 'ms') without depending on the API's verbose one. + const details = { + ...summary, + rawUnit: summary.unit || null, + unit, + warnThresh, + errorThresh, + standardLabel: standardLabel || null, + worst, + badSampleCount: bad, + badSamplePct: badPct, + appName, + // SCM UI deep-link. The renderer surfaces this as a + // "View in Prisma UI" markdown link so an operator can jump from + // the diag output straight to the Prisma UI dashboard for the + // exact site+app combination without hunting for it. + detailsUrl: audio.detailsUrl || null, + }; + + const worstStr = worst == null ? '—' : `${worst}${unit}`; + const avgStr = summary.avg == null ? '—' : `${summary.avg}${unit}`; + const cmp = lowIsBad ? '<' : '>'; + const threshStr = status === 'error' + ? `${cmp} ${errorThresh}${unit}` + : `${cmp} ${warnThresh}${unit}`; + + if (status === 'error' || status === 'warn') { + return { + status, + message: + `Actual voice traffic (${appName}) worst-window ${label}: ${worstStr} ${threshStr} ` + + `(avg ${avgStr} over ${summary.validSamples} × ${summary.interval || '5min'} samples). ` + + `${badPct}% of samples were ${status === 'error' ? 'in the error range' : 'past warn'}. ` + + `This is measured on real RTP traffic — the WAN link probes may still look fine.`, + details, + remediation: null, + }; + } + + return { + status: 'ok', + message: + `Actual voice traffic (${appName}) ${label} within range: worst ${worstStr}, ` + + `avg ${avgStr} across ${summary.validSamples} × ${summary.interval || '5min'} samples.`, + details, + remediation: null, + }; +} + /** * Evaluate a numeric per-link metric against warn/error thresholds * across every path, returning a CheckResult. Worst path drives diff --git a/services/voiceDiag/checks/wan/wanAppRtpJitter.js b/services/voiceDiag/checks/wan/wanAppRtpJitter.js new file mode 100644 index 0000000..0f747e8 --- /dev/null +++ b/services/voiceDiag/checks/wan/wanAppRtpJitter.js @@ -0,0 +1,57 @@ +// src/services/voiceDiag/checks/wan/wanAppRtpJitter.js +// +// Per-application audio jitter — measured on ACTUAL RTP voice +// traffic via Prisma DPI. Metric name on the v2.6 monitor/metrics +// endpoint is `AppPerfUDPAudioJitter` (unit: milliseconds), takes +// filter.direction="Ingress" AND filter.path_type=[all]. +// +// See wanAppRtpMos.js for the app-selection contract — the check is +// app-agnostic; the tenant picks which voice app to grade via env. +// +// Grades against the WORST-window jitter — same reasoning as the +// MOS + loss checks: a brief burst that ruins in-flight calls is +// exactly what needs surfacing. +// +// Skipped when the voice-app env is not configured. + +import { + maybeSkippedByKillSwitch, + getAppAudio, + evaluateAppAudioMetric, + getAppJitterWarnMs, + getAppJitterErrorMs, +} from './_helpers.js'; + +export const WAN_APP_RTP_JITTER_STANDARDS = Object.freeze({ + maxWarnMs: 30, + maxErrorMs: 50, + unit: 'ms', + reference: 'RFC 3550 (jitter buffer tolerance)', + gradeAgainst: 'worst-window', +}); + +export const wanAppRtpJitterCheck = { + id: 'wanAppRtpJitter', + label: 'SD-WAN Voice Traffic Jitter (worst window)', + requires: ['sdwanSite'], + scope: null, + standards: WAN_APP_RTP_JITTER_STANDARDS, + + async run(ctx) { + const skip = maybeSkippedByKillSwitch(wanAppRtpJitterCheck); + if (skip) return skip; + + const audio = getAppAudio(ctx, 'jitter'); + const warnThresh = getAppJitterWarnMs(); + const errorThresh = getAppJitterErrorMs(); + return evaluateAppAudioMetric({ + audio, + label: 'jitter', + unit: 'ms', + warnThresh, + errorThresh, + lowIsBad: false, + standardLabel: `warn > ${warnThresh}ms, error > ${errorThresh}ms`, + }); + }, +}; diff --git a/services/voiceDiag/checks/wan/wanAppRtpLoss.js b/services/voiceDiag/checks/wan/wanAppRtpLoss.js new file mode 100644 index 0000000..4e9e8fe --- /dev/null +++ b/services/voiceDiag/checks/wan/wanAppRtpLoss.js @@ -0,0 +1,58 @@ +// src/services/voiceDiag/checks/wan/wanAppRtpLoss.js +// +// Per-application audio packet loss — measured on ACTUAL RTP voice +// traffic via Prisma DPI. Metric name on the v2.6 monitor/metrics +// endpoint is `AppPerfUDPAudioPacketLoss` (unit: percentage), takes +// filter.direction="Ingress" AND filter.path_type=[all]. +// +// See wanAppRtpMos.js for the app-selection contract — the check is +// app-agnostic; the tenant picks which voice app to grade via env. +// +// Grades against the WORST-window loss in the series. A 5-minute +// burst of 20% loss makes calls unusable during that window even +// if the 24h avg is 2% — always show the operator the worst-case +// number so they know a bad experience is real, not aggregated away. +// +// Skipped when the voice-app env is not configured. + +import { + maybeSkippedByKillSwitch, + getAppAudio, + evaluateAppAudioMetric, + getAppLossWarnPct, + getAppLossErrorPct, +} from './_helpers.js'; + +export const WAN_APP_RTP_LOSS_STANDARDS = Object.freeze({ + maxWarnPct: 5, + maxErrorPct: 15, + unit: '%', + reference: 'ITU-T G.113 (voice loss tolerance)', + gradeAgainst: 'worst-window', +}); + +export const wanAppRtpLossCheck = { + id: 'wanAppRtpLoss', + label: 'SD-WAN Voice Traffic Packet Loss (worst window)', + requires: ['sdwanSite'], + scope: null, + standards: WAN_APP_RTP_LOSS_STANDARDS, + + async run(ctx) { + const skip = maybeSkippedByKillSwitch(wanAppRtpLossCheck); + if (skip) return skip; + + const audio = getAppAudio(ctx, 'loss'); + const warnThresh = getAppLossWarnPct(); + const errorThresh = getAppLossErrorPct(); + return evaluateAppAudioMetric({ + audio, + label: 'packet loss', + unit: '%', + warnThresh, + errorThresh, + lowIsBad: false, + standardLabel: `warn > ${warnThresh}%, error > ${errorThresh}%`, + }); + }, +}; diff --git a/services/voiceDiag/checks/wan/wanAppRtpMos.js b/services/voiceDiag/checks/wan/wanAppRtpMos.js new file mode 100644 index 0000000..0466a3d --- /dev/null +++ b/services/voiceDiag/checks/wan/wanAppRtpMos.js @@ -0,0 +1,65 @@ +// src/services/voiceDiag/checks/wan/wanAppRtpMos.js +// +// Per-application audio MOS — measured on ACTUAL RTP voice traffic +// via Prisma DPI, not on synthetic link probes. Metric name on the +// v2.6 monitor/metrics endpoint is `AppAudioMos` (unit: count), +// requires filter.direction="Ingress". +// +// Which app is measured depends on the tenant's configured voice +// application (PRISMA_APP_ID_VOICE env). Common choices are +// `Webex_Calling_RTP` (recommended for Webex Calling shops — the +// Webex-specific DPI signature excludes noise from other UDP +// traffic that pollutes generic RTP), `rtp-base`, `MS_Teams_RTP`, +// etc. The check is app-agnostic; the app name is threaded through +// the message + details so operators know which app was graded. +// +// Grades against the WORST-window MOS in the series. A store that +// averages 4.0 MOS over 24h but dipped to 1.85 for several 5-min +// windows had unusable calls in those windows — the average smooths +// that away and the LQM link-probe average smooths it away twice. +// This check is the escape hatch for that failure mode. +// +// Skipped when the voice-app env is not configured (feature-gated +// to keep the extra Prisma API load opt-in per tenant). + +import { + maybeSkippedByKillSwitch, + getAppAudio, + evaluateAppAudioMetric, + getAppMosWarn, + getAppMosError, +} from './_helpers.js'; + +export const WAN_APP_RTP_MOS_STANDARDS = Object.freeze({ + minWarnMos: 4.0, + minErrorMos: 3.5, + unit: 'count', + reference: 'ITU-T P.800 (MOS)', + gradeAgainst: 'worst-window', +}); + +export const wanAppRtpMosCheck = { + id: 'wanAppRtpMos', + label: 'SD-WAN Voice Traffic MOS (worst window)', + requires: ['sdwanSite'], + scope: null, + standards: WAN_APP_RTP_MOS_STANDARDS, + + async run(ctx) { + const skip = maybeSkippedByKillSwitch(wanAppRtpMosCheck); + if (skip) return skip; + + const audio = getAppAudio(ctx, 'mos'); + const warnThresh = getAppMosWarn(); + const errorThresh = getAppMosError(); + return evaluateAppAudioMetric({ + audio, + label: 'MOS', + unit: '', + warnThresh, + errorThresh, + lowIsBad: true, + standardLabel: `warn < ${warnThresh}, error < ${errorThresh}`, + }); + }, +}; diff --git a/tests/alarmSemantics.test.js b/tests/alarmSemantics.test.js index 36ad9f4..72cb096 100644 --- a/tests/alarmSemantics.test.js +++ b/tests/alarmSemantics.test.js @@ -170,10 +170,12 @@ test('humanizeAge: missing / invalid ts → empty string', () => { // ─── humanizeWindow ────────────────────────────────────────────────── test('humanizeWindow: canonical values', () => { - assert.equal(humanizeWindow(15), '15m'); - assert.equal(humanizeWindow(45), '45m'); - assert.equal(humanizeWindow(60), '1h'); - assert.equal(humanizeWindow(360), '6h'); - assert.equal(humanizeWindow(1440), '1d'); - assert.equal(humanizeWindow(2880), '2d'); + assert.equal(humanizeWindow(15), '15m'); + assert.equal(humanizeWindow(45), '45m'); + assert.equal(humanizeWindow(60), '1h'); + assert.equal(humanizeWindow(360), '6h'); + assert.equal(humanizeWindow(1440), '1d'); + assert.equal(humanizeWindow(2880), '2d'); + assert.equal(humanizeWindow(10080), '7d', + 'the new default WAN window must round-trip cleanly'); }); diff --git a/tests/paloalto.client.test.js b/tests/paloalto.client.test.js index 51749b4..076c482 100644 --- a/tests/paloalto.client.test.js +++ b/tests/paloalto.client.test.js @@ -15,7 +15,13 @@ import test from 'node:test'; import assert from 'node:assert/strict'; import http from 'node:http'; -import { paloAltoAxios, getPrismaToken, _resetPrismaAuthCache } from '../integrations/paloalto/client.js'; +import { + paloAltoAxios, + getPrismaToken, + _resetPrismaAuthCache, + _resetPrismaConcurrency, + _prismaInflightCount, +} from '../integrations/paloalto/client.js'; const FAKE_TOKEN_1 = 'token-round-1'; const FAKE_TOKEN_2 = 'token-round-2'; @@ -54,9 +60,16 @@ async function makeFakeAuthServer(handlers = {}) { } const handler = explicitHandler; if (handler) { - const result = handler({ req, body }); - res.writeHead(result.status || 200, { 'Content-Type': 'application/json' }); - res.end(JSON.stringify(result.body || {})); + // Handlers may return a Promise so a test can pause the + // response until an external gate resolves — used by the + // concurrency-limiter test to hold in-flight requests open. + Promise.resolve(handler({ req, body })).then((result) => { + res.writeHead(result.status || 200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify(result.body || {})); + }).catch((err) => { + res.writeHead(500); + res.end(JSON.stringify({ err: err.message })); + }); return; } res.writeHead(404); @@ -324,3 +337,114 @@ test('client: axios instance retries once after 401 with forced refresh', async _resetPrismaAuthCache(); } }); + +// ─── Concurrency limiter ─────────────────────────────────────────── +// +// Regression guard for the /voicediag rate-limit cascade: without +// the semaphore, fanning out 10 metric calls in parallel would blow +// past Prisma's per-second cap and 429 half of them. The cap has to +// bound concurrent in-flight AXIOS calls at MAX_INFLIGHT (default 3) +// AND has to release on both success + failure paths so 429 retries +// don't stall queued callers. +// +// Testing strategy: the server introduces a small artificial delay +// so we can observe the peak concurrent request count. All +// responses complete before the test's finally block, so no socket +// races on server-close. + +test('client: concurrency limiter caps peak concurrent server-side requests at MAX_INFLIGHT=3', async () => { + _resetPrismaAuthCache(); + _resetPrismaConcurrency(); + clearAllEnv(); + + let peakConcurrent = 0; + let concurrent = 0; + + const fake = await makeFakeAuthServer({ + 'POST /oauth2/access_token': () => ({ + status: 200, body: { access_token: FAKE_TOKEN_1, expires_in: 900 }, + }), + // ~80ms delay per response — long enough to let the client fill + // its 3-permit window, short enough that all 8 requests finish + // in ~250-400ms without complex gating logic. + 'GET /slow': async () => { + concurrent += 1; + if (concurrent > peakConcurrent) peakConcurrent = concurrent; + await new Promise((r) => setTimeout(r, 80)); + concurrent -= 1; + return { status: 200, body: { ok: true } }; + }, + }); + process.env.PRISMA_AUTH_MODE = 'sase'; + process.env.PRISMA_SASE_BASE_URL = fake.baseUrl; + process.env.PRISMA_AUTH_URL = `${fake.baseUrl}/oauth2/access_token`; + process.env.PRISMA_CLIENT_ID = 'id'; + process.env.PRISMA_CLIENT_SECRET = 'secret'; + process.env.PRISMA_TSG_ID = 'tsg'; + + try { + const results = await Promise.all( + Array.from({ length: 8 }, () => paloAltoAxios.get('/slow')), + ); + assert.equal(results.length, 8); + assert.ok( + peakConcurrent <= 3, + `peak concurrent server-side requests should be <= 3 (semaphore cap), was ${peakConcurrent}`, + ); + // If the cap works, the first batch of 3 completes ~80ms in, + // and the next batch fills — peak MUST equal MAX_INFLIGHT under + // any realistic scheduling. If it's under 3, the semaphore is + // too tight or requests are strictly serial (which would also + // be a bug). + assert.ok(peakConcurrent >= 2, + `expected the client to actually parallelize (peak >= 2), was ${peakConcurrent}`); + assert.equal(_prismaInflightCount(), 0, + 'inflight counter must return to 0 after all requests finish'); + } finally { + await fake.close(); + clearAllEnv(); + _resetPrismaAuthCache(); + _resetPrismaConcurrency(); + } +}); + +test('client: concurrency permit released on 500 (failure path drains cleanly, no permit leak)', async () => { + _resetPrismaAuthCache(); + _resetPrismaConcurrency(); + clearAllEnv(); + + let apiCalls = 0; + const fake = await makeFakeAuthServer({ + 'POST /oauth2/access_token': () => ({ + status: 200, body: { access_token: FAKE_TOKEN_1, expires_in: 900 }, + }), + 'GET /explode': () => { apiCalls += 1; return { status: 500, body: { err: 'boom' } }; }, + }); + process.env.PRISMA_AUTH_MODE = 'sase'; + process.env.PRISMA_SASE_BASE_URL = fake.baseUrl; + process.env.PRISMA_AUTH_URL = `${fake.baseUrl}/oauth2/access_token`; + process.env.PRISMA_CLIENT_ID = 'id'; + process.env.PRISMA_CLIENT_SECRET = 'secret'; + process.env.PRISMA_TSG_ID = 'tsg'; + + try { + // 6 requests that all 500. If the permit isn't released on the + // error path, the 4th onwards would hang forever — the outer + // 20s axios timeout would fire and the test would fail with + // timeout rather than clean rejections. All should reject + // cleanly and inflight must return to zero. + const results = await Promise.allSettled( + Array.from({ length: 6 }, () => paloAltoAxios.get('/explode')), + ); + assert.equal(results.filter((r) => r.status === 'rejected').length, 6, + 'all 6 requests should have rejected — none hung'); + assert.equal(apiCalls, 6, 'all 6 reached the server (queue drained)'); + assert.equal(_prismaInflightCount(), 0, + 'inflight counter should return to 0 after failure — no permit leak'); + } finally { + await fake.close(); + clearAllEnv(); + _resetPrismaAuthCache(); + _resetPrismaConcurrency(); + } +}); diff --git a/tests/paloalto.metrics.test.js b/tests/paloalto.metrics.test.js index a0aa6ac..c5a499a 100644 --- a/tests/paloalto.metrics.test.js +++ b/tests/paloalto.metrics.test.js @@ -13,7 +13,9 @@ import http from 'node:http'; import { getHealthscore, getLqmMetric, + getAppMetric, getAlarms, + APP_METRIC_NAMES, } from '../integrations/paloalto/metrics.js'; import { _resetPrismaAuthCache } from '../integrations/paloalto/client.js'; @@ -329,3 +331,214 @@ test('getAlarms: uses events/query endpoint (not /monitor/alarms which 404s)', a await fake.close(); _resetPrismaAuthCache(); clearEnv(); } }); + +// ─── Per-application metrics — body-shape regressions ────────────── +// +// Ground truth for these assertions comes from two HAR captures +// against a live tenant (site CG00127), both taken 2026-07-09: +// - /Users/McqueenJ/Downloads/rtp-base-metricCG00127.har +// (app id 15932000365560116) +// - /Users/McqueenJ/Downloads/webex-base-metricCG00127.har +// (app id 1708539371717015196 → Webex_Calling_RTP) +// +// The metric names, units, and filter shapes are IDENTICAL across +// voice apps — only filter.app differs. That's why the check code is +// app-agnostic and these tests use `` as a filler rather than +// pinning one app. +// +// Every non-obvious body key here is one Prisma silently rejects if +// wrong. Do NOT relax these without a fresh HAR — see +// scripts/prismaProbe.js `try-shapes app-audio-*` to re-discover. + +test('getAppMetric(mos): AppAudioMos requires direction="Ingress" and OMITS filter.path_type', async () => { + _resetPrismaAuthCache(); + let seenBody = null; + const fake = await makeFakePrisma({ + 'POST /sdwan/monitor/v2.6/api/monitor/metrics': ({ body }) => { + seenBody = body; + return { body: { metrics: [{ series: [{ name: 'AppAudioMos', unit: 'count', data: [{ datapoints: [] }] }] }] } }; + }, + }); + setSaseEnv(fake.baseUrl); + try { + await getAppMetric('site-CG127', '15932000365560116', 'mos', 60); + assert.ok(seenBody, 'request reached fake server'); + + // The winning metric name from the HAR — anything else 400s + // with METRIC_NOT_SUPPORTED, so pin it. + assert.deepEqual(seenBody.metrics, [{ + name: 'AppAudioMos', statistics: ['average'], unit: 'count', + }]); + + // AppAudioMos is the ONE audio metric that does NOT take + // filter.path_type. If we add it, the schema check flips from + // "no matching metric" to "unsupported filter" — different bug + // symptom, but same net "returns nothing" outcome. + assert.ok(!('path_type' in seenBody.filter), + 'AppAudioMos must NOT include filter.path_type (unlike AppPerfUDP*)'); + + // Direction is REQUIRED. Without it Prisma returns 400. + assert.equal(seenBody.filter.direction, 'Ingress', + 'AppAudioMos requires filter.direction="Ingress" (audio quality is what you RECEIVE)'); + + assert.deepEqual(seenBody.filter.site, ['site-CG127']); + assert.deepEqual(seenBody.filter.app, ['15932000365560116'], + 'app id must be sent as a STRING inside an ARRAY, not a bare int or a bare string'); + + // Empty view {} — the single-metric per-app queries use this + // rather than {summary:true} or {individual:'app'}. + assert.deepEqual(seenBody.view, {}); + + assert.ok(seenBody.start_time && seenBody.end_time, + 'start_time + end_time both required — v2.6 monitor/metrics rejects without them'); + } finally { + await fake.close(); _resetPrismaAuthCache(); clearEnv(); + } +}); + +test('getAppMetric(loss): AppPerfUDPAudioPacketLoss includes ALL 5 path_types', async () => { + _resetPrismaAuthCache(); + let seenBody = null; + const fake = await makeFakePrisma({ + 'POST /sdwan/monitor/v2.6/api/monitor/metrics': ({ body }) => { + seenBody = body; + return { body: { metrics: [{ series: [{ name: 'AppPerfUDPAudioPacketLoss', unit: 'percentage', data: [{ datapoints: [] }] }] }] } }; + }, + }); + setSaseEnv(fake.baseUrl); + try { + await getAppMetric('site-CG127', '15932000365560116', 'loss', 60); + assert.equal(seenBody.metrics[0].name, 'AppPerfUDPAudioPacketLoss'); + // Case-sensitive — 'Percentage' returned METRIC_UNIT_NOT_SUPPORTED + // in earlier probe runs. Lowercase-p is a live constraint. + assert.equal(seenBody.metrics[0].unit, 'percentage'); + assert.equal(seenBody.filter.direction, 'Ingress'); + + const paths = seenBody.filter.path_type; + assert.ok(Array.isArray(paths)); + for (const t of ['DirectInternet', 'VPN', 'PrivateWAN', 'PrivateVPN', 'ServiceLink']) { + assert.ok(paths.includes(t), `path_type must include "${t}" (Prisma UI passes all 5)`); + } + } finally { + await fake.close(); _resetPrismaAuthCache(); clearEnv(); + } +}); + +test('getAppMetric(jitter): AppPerfUDPAudioJitter uses milliseconds unit', async () => { + _resetPrismaAuthCache(); + let seenBody = null; + const fake = await makeFakePrisma({ + 'POST /sdwan/monitor/v2.6/api/monitor/metrics': ({ body }) => { + seenBody = body; + return { body: {} }; + }, + }); + setSaseEnv(fake.baseUrl); + try { + await getAppMetric('site-CG127', '15932000365560116', 'jitter', 60); + assert.equal(seenBody.metrics[0].name, 'AppPerfUDPAudioJitter'); + assert.equal(seenBody.metrics[0].unit, 'milliseconds'); + assert.equal(seenBody.filter.direction, 'Ingress'); + assert.ok(Array.isArray(seenBody.filter.path_type)); + } finally { + await fake.close(); _resetPrismaAuthCache(); clearEnv(); + } +}); + +test('getAppMetric(bandwidth): AppPerfUDPAudioBandwidth has NO direction (bidirectional aggregate)', async () => { + _resetPrismaAuthCache(); + let seenBody = null; + const fake = await makeFakePrisma({ + 'POST /sdwan/monitor/v2.6/api/monitor/metrics': ({ body }) => { + seenBody = body; + return { body: {} }; + }, + }); + setSaseEnv(fake.baseUrl); + try { + await getAppMetric('site-CG127', '15932000365560116', 'bandwidth', 60); + assert.equal(seenBody.metrics[0].name, 'AppPerfUDPAudioBandwidth'); + assert.equal(seenBody.metrics[0].unit, 'Mbps'); + assert.ok(!('direction' in seenBody.filter), + 'bandwidth is a bidirectional aggregate — Prisma rejects filter.direction on this metric'); + assert.ok(Array.isArray(seenBody.filter.path_type)); + } finally { + await fake.close(); _resetPrismaAuthCache(); clearEnv(); + } +}); + +test('getAppMetric: no-op with clean null return when siteId or appId missing', async () => { + // No network setup — if the wrapper tried to make a call, the + // client would fail to resolve the SASE URL and throw. A silent + // null keeps the enrichment pipeline resilient when a caller + // forgets to configure PRISMA_APP_ID_VOICE (or the legacy + // PRISMA_APP_ID_RTP_BASE). + const r1 = await getAppMetric(null, '1708539371717015196', 'mos'); + const r2 = await getAppMetric('site-X', null, 'mos'); + assert.equal(r1, null); + assert.equal(r2, null); +}); + +test('getAppMetric: uses 5min interval for 24h window (NOT 1day — preserves 288-point time series)', async () => { + // The whole point of client-side aggregation is that we can + // extract WORST-window numbers from the series. If we ask for + // interval=1day on a 1440min window, Prisma returns 1-2 + // aggregated datapoints and the min/max collapse to the same + // value as the avg — completely defeating the purpose. HAR + // confirms the Prisma UI uses 5min for the 24h view. + _resetPrismaAuthCache(); + let seenBody = null; + const fake = await makeFakePrisma({ + 'POST /sdwan/monitor/v2.6/api/monitor/metrics': ({ body }) => { + seenBody = body; + return { body: {} }; + }, + }); + setSaseEnv(fake.baseUrl); + try { + await getAppMetric('site-CG127', '15932000365560116', 'mos', 1440); + assert.equal(seenBody.interval, '5min', + '1440-min window MUST use 5min interval (yields ~288 pts, matches HAR). ' + + 'Falling back to 1day would collapse the series to 1-2 aggregated pts.'); + } finally { + await fake.close(); _resetPrismaAuthCache(); clearEnv(); + } +}); + +test('getAppMetric: uses 1hour interval for 7d window (168 points — the new default)', async () => { + _resetPrismaAuthCache(); + let seenBody = null; + const fake = await makeFakePrisma({ + 'POST /sdwan/monitor/v2.6/api/monitor/metrics': ({ body }) => { + seenBody = body; + return { body: {} }; + }, + }); + setSaseEnv(fake.baseUrl); + try { + // 7 days = 10080 min. This is now the default window (widened + // from 24h so sporadic Webex Calling stores get enough per-app + // samples). 5min buckets would be 2016 points per metric × 4 + // metrics = 8064 numbers per site request. Snap to 1hour (168 + // pts) to keep payloads reasonable while still catching + // worst-hour degradation. + await getAppMetric('site-CG127', '1708539371717015196', 'mos', 10080); + assert.equal(seenBody.interval, '1hour'); + } finally { + await fake.close(); _resetPrismaAuthCache(); clearEnv(); + } +}); + +test('APP_METRIC_NAMES: registry exposes the HAR-confirmed winners', () => { + // Cheap "someone renamed the constant" check. The four keys are + // the API surface that sdwanEnrichment.js iterates over — a rename + // would silently drop that metric from the enrichment payload + // (parseAppSeries would return null and the check would go to + // skipped rather than error). + assert.equal(APP_METRIC_NAMES.mos.name, 'AppAudioMos'); + assert.equal(APP_METRIC_NAMES.loss.name, 'AppPerfUDPAudioPacketLoss'); + assert.equal(APP_METRIC_NAMES.jitter.name, 'AppPerfUDPAudioJitter'); + assert.equal(APP_METRIC_NAMES.bandwidth.name, 'AppPerfUDPAudioBandwidth'); + assert.equal(APP_METRIC_NAMES.mos.lowIsBad, true, + 'MOS grades against worst-window minimum — lowIsBad must be true'); +}); diff --git a/tests/paloalto.urls.test.js b/tests/paloalto.urls.test.js new file mode 100644 index 0000000..87111da --- /dev/null +++ b/tests/paloalto.urls.test.js @@ -0,0 +1,83 @@ +// tests/paloalto.urls.test.js +// +// Pure unit tests for integrations/paloalto/urls.js — no HTTP, no +// mocks. The builders here feed deep-link markdown into user-facing +// output, so any silent format change would show up as broken links +// in production. The tests spell out both the exact URL shape and +// the boundary conditions (null ids, base-URL env override). + +import test from 'node:test'; +import assert from 'node:assert/strict'; + +import { + getPrismaUiBaseUrl, + buildAppDetailsUrl, +} from '../integrations/paloalto/urls.js'; + +function withEnv(kvs, fn) { + const prev = {}; + for (const k of Object.keys(kvs)) prev[k] = process.env[k]; + try { + for (const [k, v] of Object.entries(kvs)) { + if (v === undefined) delete process.env[k]; + else process.env[k] = v; + } + return fn(); + } finally { + for (const [k, v] of Object.entries(prev)) { + if (v === undefined) delete process.env[k]; + else process.env[k] = v; + } + } +} + +test('getPrismaUiBaseUrl: defaults to the production SCM hostname', () => { + withEnv({ PRISMA_UI_BASE_URL: undefined }, () => { + assert.equal(getPrismaUiBaseUrl(), 'https://stratacloudmanager.paloaltonetworks.com'); + }); +}); + +test('getPrismaUiBaseUrl: PRISMA_UI_BASE_URL override is honored', () => { + withEnv({ PRISMA_UI_BASE_URL: 'https://scm.partner.example.com' }, () => { + assert.equal(getPrismaUiBaseUrl(), 'https://scm.partner.example.com'); + }); +}); + +test('getPrismaUiBaseUrl: trailing slashes are stripped so `${base}/x` concats cleanly', () => { + // A trailing slash would produce "//insights/…" which some routers + // treat as a different route from "/insights/…". Normalize. + withEnv({ PRISMA_UI_BASE_URL: 'https://scm.example.com///' }, () => { + assert.equal(getPrismaUiBaseUrl(), 'https://scm.example.com'); + }); +}); + +test('getPrismaUiBaseUrl: empty / whitespace env falls back to default', () => { + withEnv({ PRISMA_UI_BASE_URL: ' ' }, () => { + assert.equal(getPrismaUiBaseUrl(), 'https://stratacloudmanager.paloaltonetworks.com'); + }); +}); + +test('buildAppDetailsUrl: builds the exact SCM Application Path Details URL', () => { + // Ground truth from the user (CG00127 + rtp-base app). + withEnv({ PRISMA_UI_BASE_URL: undefined }, () => { + const url = buildAppDetailsUrl('16190109915660160', '15932000365560116'); + assert.equal( + url, + 'https://stratacloudmanager.paloaltonetworks.com/insights/operational/sdwan-applications/15932000365560116/site/16190109915660160/details', + ); + }); +}); + +test('buildAppDetailsUrl: null / missing ids → null (renderers no-op on falsy)', () => { + assert.equal(buildAppDetailsUrl(null, '15932000365560116'), null); + assert.equal(buildAppDetailsUrl('16190109915660160', null), null); + assert.equal(buildAppDetailsUrl(null, null), null); + assert.equal(buildAppDetailsUrl('', ''), null); +}); + +test('buildAppDetailsUrl: numeric ids coerced to strings in the URL (Prisma ids are int-like)', () => { + withEnv({ PRISMA_UI_BASE_URL: 'https://scm.example.com' }, () => { + const url = buildAppDetailsUrl(16190109915660160n, 15932000365560116n); + assert.match(url, /15932000365560116\/site\/16190109915660160\/details$/); + }); +}); diff --git a/tests/renderers.wan.test.js b/tests/renderers.wan.test.js index dc794eb..5ac137a 100644 --- a/tests/renderers.wan.test.js +++ b/tests/renderers.wan.test.js @@ -168,3 +168,114 @@ test('renderer: healthscore missing → "n/a" with unknown icon', () => { const md = renderWanDiagnosticsMarkdown(baseData({ healthscore: null }), { storeNum: '782' }); assert.match(md, /Healthscore.*n\/a/); }); + +// ─── Per-app "Voice Traffic Quality" section ──────────────────────── + +function appAudioSummary({ values, unit = '', interval = '5min' } = {}) { + const nums = values.filter((v) => Number.isFinite(v)); + const avg = nums.length ? Math.round((nums.reduce((a, b) => a + b, 0) / nums.length) * 100) / 100 : null; + return { + unit, interval, + samples: values.length, validSamples: nums.length, + avg, + min: nums.length ? Math.min(...nums) : null, + max: nums.length ? Math.max(...nums) : null, + p95: nums.length ? nums.sort((a, b) => a - b)[Math.floor(0.95 * nums.length)] || nums[nums.length - 1] : null, + values, + }; +} + +test('renderer: renders Voice Traffic Quality section when appAudio present with samples', () => { + // Uses Webex_Calling_RTP as the configured app — the renderer must + // pick up appName from the fixture, not hardcode "rtp-base". + const data = baseData({ + appAudio: { + appId: '1708539371717015196', appName: 'Webex_Calling_RTP', + mos: appAudioSummary({ values: [3.55, 3.55, 1.85, 4.41], unit: 'count' }), + loss: appAudioSummary({ values: [22, 0, 69, 0], unit: '%' }), + jitter: appAudioSummary({ values: [0, 0, 0], unit: 'ms' }), + bandwidth: appAudioSummary({ values: [0.5, 0.6, 0.4], unit: 'Mbps' }), + }, + }); + const md = renderWanDiagnosticsMarkdown(data, { storeNum: '127' }); + assert.match(md, /Voice Traffic Quality \(Webex_Calling_RTP/, + 'section header must reflect the tenant-configured app name'); + assert.match(md, /DPI/, 'callout for "real DPI measurement" surfaces'); + // Worst-window numbers must appear (that's the whole point). + assert.match(md, /1\.85/, 'worst MOS surfaced'); + assert.match(md, /69%/, 'worst loss surfaced'); +}); + +test('renderer: still renders correctly with legacy "rtp-base" appName (backwards-compat)', () => { + // Tenants that stayed on the legacy PRISMA_APP_ID_RTP_BASE env get + // appName='rtp-base' from resolveVoiceAppConfig() — must render + // identically to any other app. + const data = baseData({ + appAudio: { + appId: '15932000365560116', appName: 'rtp-base', + mos: appAudioSummary({ values: [4.35], unit: 'count' }), + }, + }); + const md = renderWanDiagnosticsMarkdown(data, { storeNum: '127' }); + assert.match(md, /Voice Traffic Quality \(rtp-base/); +}); + +test('renderer: omits Voice Traffic Quality section entirely when appAudio is null', () => { + const md = renderWanDiagnosticsMarkdown(baseData(), { storeNum: '782' }); + assert.doesNotMatch(md, /Voice Traffic Quality/); +}); + +test('renderer: omits Voice Traffic Quality section when appAudio has no valid samples', () => { + const empty = appAudioSummary({ values: [null, null] }); + const data = baseData({ + appAudio: { appId: 'x', appName: 'Webex_Calling_RTP', mos: empty, loss: empty, jitter: empty, bandwidth: empty }, + }); + const md = renderWanDiagnosticsMarkdown(data, { storeNum: '782' }); + assert.doesNotMatch(md, /Voice Traffic Quality/, + 'no data across every metric → skip the header entirely rather than show empty rows'); +}); + +test('renderer: MOS row uses worst-lowest label (not worst-highest)', () => { + const data = baseData({ + appAudio: { + appId: 'x', appName: 'Webex_Calling_RTP', + mos: appAudioSummary({ values: [4.4, 4.3, 3.9], unit: '' }), + }, + }); + const md = renderWanDiagnosticsMarkdown(data, { storeNum: '782' }); + assert.match(md, /worst \(lowest\)/, 'MOS is lowIsBad — must say "lowest" not "highest"'); +}); + +test('renderer: "View in Prisma UI" deep link renders when appAudio.detailsUrl is present', () => { + const data = baseData({ + appAudio: { + appId: '1708539371717015196', appName: 'Webex_Calling_RTP', + detailsUrl: 'https://stratacloudmanager.paloaltonetworks.com/insights/operational/sdwan-applications/1708539371717015196/site/16190109915660160/details', + mos: appAudioSummary({ values: [4.35] }), + }, + }); + const md = renderWanDiagnosticsMarkdown(data, { storeNum: '127' }); + assert.match( + md, + /\[View in Prisma UI\]\(https:\/\/stratacloudmanager\.paloaltonetworks\.com\/insights\/operational\/sdwan-applications\/1708539371717015196\/site\/16190109915660160\/details\)/, + 'deep link must be rendered as a markdown link inside the section header', + ); +}); + +test('renderer: no deep link when appAudio.detailsUrl is missing (defensive)', () => { + const data = baseData({ + appAudio: { + appId: 'x', appName: 'Webex_Calling_RTP', + mos: appAudioSummary({ values: [4.35] }), + }, + }); + const md = renderWanDiagnosticsMarkdown(data, { storeNum: '127' }); + assert.doesNotMatch(md, /View in Prisma UI/, + 'link line should be omitted entirely when the URL is null'); +}); + +test('renderer: footer mentions the per-app --only shortcut', () => { + const md = renderWanDiagnosticsMarkdown(baseData(), { storeNum: '782', footer: true }); + assert.match(md, /wanAppRtpMos,wanAppRtpLoss,wanAppRtpJitter/, + 'footer should point operators at the per-app checks by id'); +}); diff --git a/tests/sdwanEnrichment.test.js b/tests/sdwanEnrichment.test.js index 0a5e6db..e89e9e7 100644 --- a/tests/sdwanEnrichment.test.js +++ b/tests/sdwanEnrichment.test.js @@ -17,6 +17,9 @@ import { parseHealthscore, buildLinkRows, parseAlarms, + summarizeAppSeries, + resolveVoiceAppConfig, + _resetVoiceAppDeprecationFlag, } from '../services/enrichment/sdwanEnrichment.js'; import { _resetSitesCache } from '../integrations/paloalto/sites.js'; import { _resetPrismaAuthCache } from '../integrations/paloalto/client.js'; @@ -768,26 +771,36 @@ test('collectSdwanForStore: reports the effective window on the returned payload }); setupSaseEnv(fake.baseUrl); try { - // Default (no opts): 24h — the shipping default optimised for - // "was the site healthy today" over "is it healthy right now". - // Override via WAN_STANDARD_WINDOW_MINUTES or --window on /voicediag. + // Default (no opts): 7 days (10080 min). Widened from 24h because + // per-app DPI metrics only get datapoints when calls actually + // happen — sporadic Webex Calling stores need a wider window for + // worst-window statistics to be meaningful. Override via + // WAN_STANDARD_WINDOW_MINUTES env or --window on /voicediag. let data = await collectSdwanForStore(782); - assert.equal(data.window?.minutes, 1440); - assert.equal(data.window?.alarmMinutes, 1440, + assert.equal(data.window?.minutes, 10080); + assert.equal(data.window?.alarmMinutes, 10080, 'alarms match the requested window when it is >= 60m'); - // Explicit 15m override → both windows go to 15/60 (alarm floor) + // Explicit 15m override → metric window narrows, alarms floor at 60m data = await collectSdwanForStore(782, { windowMinutes: 15 }); assert.equal(data.window?.minutes, 15); assert.equal(data.window?.alarmMinutes, 60, 'alarms floor at 60m even when the requested window is smaller'); - // Out-of-range values are clamped to [1, 1440] + // Explicit 24h override — the common "live triage" case + data = await collectSdwanForStore(782, { windowMinutes: 1440 }); + assert.equal(data.window?.minutes, 1440, + '24h stays 24h — no forced upgrade to 7d'); + assert.equal(data.window?.alarmMinutes, 1440); + + // Out-of-range values are clamped to [1, 10080] data = await collectSdwanForStore(782, { windowMinutes: 99999 }); - assert.equal(data.window?.minutes, 1440, 'capped at 24h'); + assert.equal(data.window?.minutes, 10080, + 'capped at 7d — Prisma downsamples per-app series past this to 1-day buckets'); data = await collectSdwanForStore(782, { windowMinutes: -5 }); - assert.equal(data.window?.minutes, 1440, 'invalid → falls to env default (24h)'); + assert.equal(data.window?.minutes, 10080, + 'invalid → falls to env default (7d, was 24h before the 2026-07-09 widen)'); } finally { await fake.close(); _resetSitesCache(); _resetPrismaAuthCache(); clearEnv(); } @@ -845,3 +858,213 @@ test('collectSdwanForStore: preserves per-metric failure in errors[]', async () await fake.close(); _resetSitesCache(); _resetPrismaAuthCache(); clearEnv(); } }); + +// ─── Unit tests: summarizeAppSeries ───────────────────────────────── +// +// App metrics come back as a real time series (usually 288 pts over +// 24h at 5-min intervals). The summarizer must extract worst-window +// stats reliably regardless of null values, empty series, and the +// v2.6 response shape's quirks. + +test('summarizeAppSeries: nil / malformed inputs → null', () => { + assert.equal(summarizeAppSeries(null, 'X'), null); + assert.equal(summarizeAppSeries({}, 'X'), null); + assert.equal(summarizeAppSeries({ metrics: [] }, 'X'), null); + assert.equal(summarizeAppSeries({ metrics: [{ series: [] }] }, 'X'), null); +}); + +test('summarizeAppSeries: HAR-shape MOS response → {avg,min,max,p95}', () => { + // Truncated version of the live rtp-base MOS response from + // rtp-base-metricCG00127.har (2026-07-09). The webex HAR + // (webex-base-metricCG00127.har) has an identical body shape — + // this single test covers both apps. + const resp = { + metrics: [{ + series: [{ + name: 'AppAudioMos', + unit: 'count', + interval: '5min', + data: [{ + statistics: 'average', + datapoints: [ + { value: 3.5569644, time: 'T1' }, + { value: 3.55610356, time: 'T2' }, + { value: 3.92861172, time: 'T3' }, + { value: 1.8523214, time: 'T4' }, // worst window from the HAR + { value: 4.4092858, time: 'T5' }, // best window + { value: 4.0257409, time: 'T6' }, + ], + }], + }], + }], + }; + const s = summarizeAppSeries(resp, 'AppAudioMos'); + assert.ok(s, 'summary produced'); + assert.equal(s.samples, 6); + assert.equal(s.validSamples, 6); + assert.equal(s.unit, 'count'); + assert.equal(s.interval, '5min'); + assert.equal(s.min, 1.85, 'worst-window MOS surfaced (the whole reason for the check)'); + assert.equal(s.max, 4.41); + // Avg ≈ 3.5548 → 3.55 after round2 + assert.equal(s.avg, 3.55); + assert.ok(Array.isArray(s.values), 'raw values retained for downstream stats'); + assert.equal(s.values.length, 6); +}); + +test('summarizeAppSeries: all-null datapoints → validSamples=0, avg/min/max null', () => { + const resp = { + metrics: [{ series: [{ + name: 'AppPerfUDPAudioJitter', unit: 'milliseconds', interval: '5min', + data: [{ datapoints: [{ value: null, time: 'T1' }, { time: 'T2' }] }], + }] }], + }; + const s = summarizeAppSeries(resp, 'AppPerfUDPAudioJitter'); + assert.equal(s.samples, 2); + assert.equal(s.validSamples, 0); + assert.equal(s.avg, null); + assert.equal(s.min, null); + assert.equal(s.max, null); +}); + +test('summarizeAppSeries: mixed null + numeric → nulls ignored in aggregates', () => { + const resp = { + metrics: [{ series: [{ + name: 'AppPerfUDPAudioPacketLoss', unit: 'percentage', interval: '5min', + data: [{ datapoints: [ + { value: null, time: 'T1' }, + { value: 22, time: 'T2' }, + { value: 0, time: 'T3' }, + { value: null, time: 'T4' }, + { value: 5, time: 'T5' }, + ] }], + }] }], + }; + const s = summarizeAppSeries(resp, 'AppPerfUDPAudioPacketLoss'); + assert.equal(s.samples, 5); + assert.equal(s.validSamples, 3); + assert.equal(s.min, 0); + assert.equal(s.max, 22); + assert.equal(s.avg, 9, '(22 + 0 + 5) / 3 = 9'); +}); + +// ─── resolveVoiceAppConfig ────────────────────────────────────────── +// +// Reads env vars at call time and returns {appId, appName} for the +// tenant-configured voice application. Two env vars are honored: +// - PRISMA_APP_ID_VOICE (canonical) + PRISMA_APP_NAME_VOICE (name) +// - PRISMA_APP_ID_RTP_BASE (legacy, backwards-compat only) +// +// Every test here restores the original env so parallel test files +// don't stomp each other, and clears the deprecation-log dedupe flag +// so the deprecation-warning test can observe fresh log behavior. + +function withVoiceEnv(overrides, fn) { + const KEYS = ['PRISMA_APP_ID_VOICE', 'PRISMA_APP_ID_RTP_BASE', 'PRISMA_APP_NAME_VOICE']; + const saved = Object.fromEntries(KEYS.map((k) => [k, process.env[k]])); + try { + for (const k of KEYS) delete process.env[k]; + for (const [k, v] of Object.entries(overrides)) { + if (v !== undefined) process.env[k] = v; + } + _resetVoiceAppDeprecationFlag(); + return fn(); + } finally { + for (const k of KEYS) { + if (saved[k] === undefined) delete process.env[k]; + else process.env[k] = saved[k]; + } + _resetVoiceAppDeprecationFlag(); + } +} + +test('resolveVoiceAppConfig: nothing set → {appId:null, appName:"voice"}', () => { + withVoiceEnv({}, () => { + const cfg = resolveVoiceAppConfig(); + assert.equal(cfg.appId, null); + assert.equal(cfg.appName, 'voice'); + }); +}); + +test('resolveVoiceAppConfig: PRISMA_APP_ID_VOICE alone → id set, default name "voice"', () => { + withVoiceEnv({ PRISMA_APP_ID_VOICE: '1708539371717015196' }, () => { + const cfg = resolveVoiceAppConfig(); + assert.equal(cfg.appId, '1708539371717015196'); + assert.equal(cfg.appName, 'voice', + 'unnamed config gets the generic "voice" label — tenants override with PRISMA_APP_NAME_VOICE'); + }); +}); + +test('resolveVoiceAppConfig: PRISMA_APP_NAME_VOICE overrides display label', () => { + // The recommended Webex Calling deployment. + withVoiceEnv({ + PRISMA_APP_ID_VOICE: '1708539371717015196', + PRISMA_APP_NAME_VOICE: 'Webex_Calling_RTP', + }, () => { + const cfg = resolveVoiceAppConfig(); + assert.equal(cfg.appId, '1708539371717015196'); + assert.equal(cfg.appName, 'Webex_Calling_RTP'); + }); +}); + +test('resolveVoiceAppConfig: legacy PRISMA_APP_ID_RTP_BASE honored, appName defaults to "rtp-base"', () => { + // Backwards-compat path — existing deployments must keep working + // without any env changes. Display name defaults to "rtp-base" so + // dashboards/screenshots that reference the old label still match. + withVoiceEnv({ PRISMA_APP_ID_RTP_BASE: '15932000365560116' }, () => { + const cfg = resolveVoiceAppConfig(); + assert.equal(cfg.appId, '15932000365560116'); + assert.equal(cfg.appName, 'rtp-base'); + }); +}); + +test('resolveVoiceAppConfig: legacy id + explicit name override → uses both', () => { + withVoiceEnv({ + PRISMA_APP_ID_RTP_BASE: '15932000365560116', + PRISMA_APP_NAME_VOICE: 'rtp', + }, () => { + const cfg = resolveVoiceAppConfig(); + assert.equal(cfg.appId, '15932000365560116'); + assert.equal(cfg.appName, 'rtp', + 'explicit PRISMA_APP_NAME_VOICE wins over the legacy-default "rtp-base"'); + }); +}); + +test('resolveVoiceAppConfig: PRISMA_APP_ID_VOICE takes precedence over legacy PRISMA_APP_ID_RTP_BASE', () => { + // Migration scenario: operator set the new env alongside the old + // one before removing the old one. New value must win. + withVoiceEnv({ + PRISMA_APP_ID_VOICE: '1708539371717015196', + PRISMA_APP_ID_RTP_BASE: '15932000365560116', + }, () => { + const cfg = resolveVoiceAppConfig(); + assert.equal(cfg.appId, '1708539371717015196', + 'canonical env var wins — legacy is only a fallback'); + assert.equal(cfg.appName, 'voice', + 'name default follows the new env, not the legacy one'); + }); +}); + +test('resolveVoiceAppConfig: whitespace-only values treated as unset', () => { + withVoiceEnv({ + PRISMA_APP_ID_VOICE: ' ', + PRISMA_APP_ID_RTP_BASE: '\t\n', + PRISMA_APP_NAME_VOICE: ' ', + }, () => { + const cfg = resolveVoiceAppConfig(); + assert.equal(cfg.appId, null); + assert.equal(cfg.appName, 'voice', + 'whitespace-only PRISMA_APP_NAME_VOICE falls back to generic default'); + }); +}); + +test('resolveVoiceAppConfig: values are trimmed (env vars often carry accidental whitespace)', () => { + withVoiceEnv({ + PRISMA_APP_ID_VOICE: ' 1708539371717015196 ', + PRISMA_APP_NAME_VOICE: ' Webex_Calling_RTP ', + }, () => { + const cfg = resolveVoiceAppConfig(); + assert.equal(cfg.appId, '1708539371717015196', 'id trimmed'); + assert.equal(cfg.appName, 'Webex_Calling_RTP', 'name trimmed'); + }); +}); diff --git a/tests/voiceDiag.checks.test.js b/tests/voiceDiag.checks.test.js index 67e77d4..6fa552e 100644 --- a/tests/voiceDiag.checks.test.js +++ b/tests/voiceDiag.checks.test.js @@ -524,13 +524,17 @@ test('parseWindowMinutes: unit-suffixed values (m/h/d) resolve to minutes', asyn assert.equal(parseWindowMinutes('24h'), 1440); assert.equal(parseWindowMinutes('1d'), 1440); assert.equal(parseWindowMinutes('1day'), 1440); + assert.equal(parseWindowMinutes('7d'), 10080, + '7d resolves to 10080 min — matches the default window and the hard cap'); + assert.equal(parseWindowMinutes('7days'), 10080); assert.equal(parseWindowMinutes('2 hours'), 120); }); test('parseWindowMinutes: bare integer treated as minutes', async () => { const { parseWindowMinutes } = await import('../commands/voiceDiag.js'); assert.equal(parseWindowMinutes('30'), 30); - assert.equal(parseWindowMinutes('1440'), 1440); + assert.equal(parseWindowMinutes('1440'), 1440); + assert.equal(parseWindowMinutes('10080'), 10080); }); test('parseWindowMinutes: empty / null → undefined (falls to env default)', async () => { diff --git a/tests/voiceDiag.wan.test.js b/tests/voiceDiag.wan.test.js index 49048fa..1149747 100644 --- a/tests/voiceDiag.wan.test.js +++ b/tests/voiceDiag.wan.test.js @@ -21,7 +21,11 @@ import { wanLatencyCheck } from '../services/voiceDiag/checks/wan/wanLatency.js' import { wanJitterCheck } from '../services/voiceDiag/checks/wan/wanJitter.js'; import { wanLossCheck } from '../services/voiceDiag/checks/wan/wanLoss.js'; import { wanMosCheck } from '../services/voiceDiag/checks/wan/wanMos.js'; +import { wanAppRtpMosCheck } from '../services/voiceDiag/checks/wan/wanAppRtpMos.js'; +import { wanAppRtpLossCheck } from '../services/voiceDiag/checks/wan/wanAppRtpLoss.js'; +import { wanAppRtpJitterCheck } from '../services/voiceDiag/checks/wan/wanAppRtpJitter.js'; import { wanAlarmsCheck } from '../services/voiceDiag/checks/wan/wanAlarms.js'; +import { humanizeMetricUnit } from '../services/voiceDiag/checks/wan/_helpers.js'; import { CHECKS } from '../services/voiceDiag/checks/index.js'; // ─── Helpers ──────────────────────────────────────────────────────── @@ -393,7 +397,7 @@ test('wanAlarms: message reflects the effective alarm window (not hardcoded 1h)' test('every WAN check exposes a standards object', () => { const wanChecks = CHECKS.filter((c) => c.id.startsWith('wan')); - assert.equal(wanChecks.length, 8, 'expected 8 registered WAN checks'); + assert.equal(wanChecks.length, 11, 'expected 11 registered WAN checks (8 link + 3 app-DPI)'); const missing = wanChecks.filter((c) => !c.standards || typeof c.standards !== 'object'); assert.deepEqual(missing.map((c) => c.id), []); }); @@ -409,9 +413,16 @@ test('WAN checks registered in the expected order after port bucket', () => { 'wanJitter', 'wanLoss', 'wanMos', + // Per-app DPI checks land after the link-probe checks — the + // link-probe pass/fail is the coarse signal, then the per-app + // check refines it. This ordering shows up in the /voicediag + // output as "link is up + green" followed by "but actual RTP + // saw..." which reads naturally for an operator. + 'wanAppRtpMos', + 'wanAppRtpLoss', + 'wanAppRtpJitter', 'wanAlarms', ]); - // WAN bucket sits after the port bucket and before phoneOnline. const portEnabledIdx = ids.indexOf('portEnabled'); const wanSiteIdx = ids.indexOf('wanSite'); const phoneOnlineIdx = ids.indexOf('phoneOnline'); @@ -425,3 +436,253 @@ test('no WAN check declares a remediation (diagnostic-only)', () => { assert.equal(c.remediations, undefined, `${c.id} should not expose remediations`); } }); + +// ─── wanAppRtp* (per-app DPI voice-quality checks) ────────────────── +// +// These grade against the WORST-window value in the series rather +// than the average — the whole point is to catch transient +// degradation the 24h link-probe average smooths away. Every case +// below spells out which value the check MUST grade against (min for +// MOS, max for loss/jitter) because if the accessor picks the wrong +// side of the summary a well-averaged store would silently pass +// while its calls sound terrible. + +function mkAppCtx({ mos, loss, jitter, bandwidth } = {}) { + const ctx = mkWanCtx({}); + // Uses Webex_Calling_RTP as the exemplar in fixtures since it's the + // recommended production choice — but the checks are app-agnostic + // so any app id/name should behave identically. + ctx.sdwanData.appAudio = { + appId: '1708539371717015196', + appName: 'Webex_Calling_RTP', + mos, loss, jitter, bandwidth, + }; + return ctx; +} + +function seriesSummary({ values, unit = '', interval = '5min' } = {}) { + // Build a shape identical to summarizeAppSeries() output so the + // check exercises the real threshold path rather than a stub. + const nums = values.filter((v) => typeof v === 'number' && Number.isFinite(v)); + let min = null, max = null, avg = null, p95 = null; + if (nums.length > 0) { + min = Math.min(...nums); + max = Math.max(...nums); + avg = Math.round((nums.reduce((a, b) => a + b, 0) / nums.length) * 100) / 100; + const sorted = [...nums].sort((a, b) => a - b); + p95 = sorted[Math.min(sorted.length - 1, Math.floor(0.95 * sorted.length))]; + } + return { + unit, interval, + samples: values.length, validSamples: nums.length, + avg, min, max, p95, values, + }; +} + +test('wanAppRtpMos: skipped when appAudio missing (feature not configured)', async () => { + const ctx = mkWanCtx({}); // no appAudio + const r = await wanAppRtpMosCheck.run(ctx); + assert.equal(r.status, 'skipped'); + // Message must point operators at the canonical env var name. + // Backwards-compat handling for the legacy PRISMA_APP_ID_RTP_BASE + // lives in resolveVoiceAppConfig(); the surface message points at + // the new name only. + assert.match(r.message, /PRISMA_APP_ID_VOICE/); +}); + +test('wanAppRtpMos: ok when worst-window MOS >= warn (default 4.0)', async () => { + const ctx = mkAppCtx({ mos: seriesSummary({ values: [4.1, 4.3, 4.5, 4.0] }) }); + const r = await wanAppRtpMosCheck.run(ctx); + assert.equal(r.status, 'ok'); +}); + +test('wanAppRtpMos: warn when worst-window MOS < 4.0 but >= 3.5 (avg irrelevant)', async () => { + // Avg = 4.05 (looks fine) but one 5-min window dipped to 3.8 → + // must warn, not pass. This is the whole reason the check exists. + const ctx = mkAppCtx({ mos: seriesSummary({ values: [4.3, 4.2, 3.8, 4.5] }) }); + const r = await wanAppRtpMosCheck.run(ctx); + assert.equal(r.status, 'warn'); + assert.match(r.message, /3\.8/, 'worst-window value must appear in message'); +}); + +test('wanAppRtpMos: error when worst-window MOS < 3.5 (matches HAR failure case)', async () => { + // Real Webex_Calling_RTP numbers from CG00127 + // (webex-base-metricCG00127.har, 2026-07-09): min=1.60, avg=4.27. + // Even though avg is fine, the min alone must fire error since + // those 5-min windows rendered calls unintelligible. + const ctx = mkAppCtx({ mos: seriesSummary({ values: [4.4, 4.3, 3.92, 1.60, 4.41, 4.03] }) }); + const r = await wanAppRtpMosCheck.run(ctx); + assert.equal(r.status, 'error'); + assert.match(r.message, /1\.6/); + assert.match(r.message, /worst-window/i); +}); + +test('wanAppRtpLoss: ok when worst-window loss <= 5%', async () => { + const ctx = mkAppCtx({ loss: seriesSummary({ values: [0, 0.5, 1, 2, 4] }) }); + const r = await wanAppRtpLossCheck.run(ctx); + assert.equal(r.status, 'ok'); +}); + +test('wanAppRtpLoss: warn when worst-window loss > 5% but <= 15%', async () => { + const ctx = mkAppCtx({ loss: seriesSummary({ values: [0, 0, 10, 0] }) }); + const r = await wanAppRtpLossCheck.run(ctx); + assert.equal(r.status, 'warn'); + assert.match(r.message, /10%/); +}); + +test('wanAppRtpLoss: error when worst-window loss > 15% (matches HAR failure)', async () => { + // Real Webex_Calling_RTP numbers from CG00127: max=26.88% loss. + // Must error even though avg was only ~1.37%. + const ctx = mkAppCtx({ loss: seriesSummary({ values: [0, 5, 26.88, 0, 3] }) }); + const r = await wanAppRtpLossCheck.run(ctx); + assert.equal(r.status, 'error'); + assert.match(r.message, /26\.88%/); +}); + +test('wanAppRtpJitter: ok when all-zero series (nothing to grade badly)', async () => { + // The HAR shows AppPerfUDPAudioJitter often returns all zeros for + // this tenant. Must be treated as "clean throughout", NOT skipped + // and NOT bad — the metric legitimately reports zero and that IS + // an ok result. + const ctx = mkAppCtx({ jitter: seriesSummary({ values: [0, 0, 0, 0, 0] }) }); + const r = await wanAppRtpJitterCheck.run(ctx); + assert.equal(r.status, 'ok'); +}); + +test('wanAppRtpJitter: error when worst-window > 50ms', async () => { + const ctx = mkAppCtx({ jitter: seriesSummary({ values: [1, 2, 75, 3, 0] }) }); + const r = await wanAppRtpJitterCheck.run(ctx); + assert.equal(r.status, 'error'); + assert.match(r.message, /75ms/); +}); + +test('wanAppRtp*: skipped when validSamples=0 (Prisma returned data but all-null)', async () => { + const emptyish = seriesSummary({ values: [null, null, null] }); + const ctx = mkAppCtx({ mos: emptyish, loss: emptyish, jitter: emptyish }); + for (const check of [wanAppRtpMosCheck, wanAppRtpLossCheck, wanAppRtpJitterCheck]) { + const r = await check.run(ctx); + assert.equal(r.status, 'skipped', `${check.id} → skipped on all-null`); + assert.match(r.message, /no voice traffic/i); + } +}); + +test('wanAppRtp*: kill switch (WAN_STANDARD_ENABLED=false) silences all three', async () => { + await withKillSwitchOn(async () => { + const ctx = mkAppCtx({ mos: seriesSummary({ values: [1.5] }) }); + for (const check of [wanAppRtpMosCheck, wanAppRtpLossCheck, wanAppRtpJitterCheck]) { + const r = await check.run(ctx); + assert.equal(r.status, 'skipped', `${check.id} kill-switch triggered`); + assert.match(r.message, /WAN_STANDARD_ENABLED=false/); + } + }); +}); + +test('wanAppRtp*: badSamplePct exposes how often the metric was in warn/error', async () => { + // 4 of 10 samples in error range → 40% bad. + const values = [0, 0, 20, 22, 0, 30, 0, 0, 50, 0]; // 4 > 15% + const ctx = mkAppCtx({ loss: seriesSummary({ values }) }); + const r = await wanAppRtpLossCheck.run(ctx); + assert.equal(r.status, 'error'); + assert.equal(r.details.badSampleCount, 4); + assert.equal(r.details.badSamplePct, 40); +}); + +test('wanAppRtp*: fetch-failure surfaces the underlying error message (not "set env var")', async () => { + // Regression for the 429-cascade UX bug: previously, a fetch + // failure with the env var CONFIGURED caused the check to say + // "set PRISMA_APP_ID_RTP_BASE" — actively misleading. Now the + // check tells the operator the real reason (429, timeout, etc). + // + // Error-scope names are app-agnostic ("app.voice.*") so this + // still routes correctly regardless of which voice app the tenant + // configured. + const ctx = mkWanCtx({}); + ctx.sdwanData.appAudio = { + appId: '1708539371717015196', + appName: 'Webex_Calling_RTP', + mos: null, // fetch failed + loss: null, // fetch failed + jitter: seriesSummary({ values: [0, 0, 0] }), // succeeded + bandwidth: seriesSummary({ values: [0.5] }), + }; + ctx.sdwanData.errors = [ + { scope: 'app.voice.mos', message: 'Request failed with status code 429 (HTTP 429)' }, + { scope: 'app.voice.loss', message: 'timeout of 20000ms exceeded' }, + ]; + + const mosResult = await wanAppRtpMosCheck.run(ctx); + assert.equal(mosResult.status, 'skipped'); + assert.match(mosResult.message, /429/, 'must surface the underlying 429'); + assert.doesNotMatch(mosResult.message, /PRISMA_APP_ID_VOICE/, + 'must NOT say "set env var" — the env IS set, the fetch just failed'); + assert.match(mosResult.message, /Retry in ~30/, 'includes actionable "try again" hint'); + + const lossResult = await wanAppRtpLossCheck.run(ctx); + assert.match(lossResult.message, /timeout/); + + // Jitter succeeded so it should grade normally, not be dragged + // down by its siblings' failures. + const jitterResult = await wanAppRtpJitterCheck.run(ctx); + assert.equal(jitterResult.status, 'ok', + 'per-metric fetch failures must not cascade into sibling metrics'); +}); + +test('humanizeMetricUnit: maps raw Prisma unit strings to concise display suffixes', () => { + // The important cases — everything we've seen in a real Prisma + // response. These are what caused "11.83percentage" / "50milliseconds" + // to leak into the UI before this was added. + assert.equal(humanizeMetricUnit('percentage'), '%'); + assert.equal(humanizeMetricUnit('percent'), '%'); + assert.equal(humanizeMetricUnit('milliseconds'), 'ms'); + assert.equal(humanizeMetricUnit('ms'), 'ms'); + assert.equal(humanizeMetricUnit('count'), '', + 'MOS unit "count" renders as empty — MOS numbers are self-explanatory'); + assert.equal(humanizeMetricUnit('gauge'), ''); + assert.equal(humanizeMetricUnit('Mbps'), 'Mbps'); + assert.equal(humanizeMetricUnit('kbps'), 'kbps'); + assert.equal(humanizeMetricUnit(''), ''); + assert.equal(humanizeMetricUnit(null), ''); + // Fallback path: unknown unit — return with a leading space so + // "12 widgets" reads better than "12widgets". + assert.equal(humanizeMetricUnit('widgets'), ' widgets'); +}); + +test('wanAppRtp*: details.detailsUrl threads the SCM deep-link from appAudio through the check', async () => { + const ctx = mkWanCtx({}); + ctx.sdwanData.appAudio = { + appId: '1708539371717015196', appName: 'Webex_Calling_RTP', + detailsUrl: 'https://stratacloudmanager.paloaltonetworks.com/insights/operational/sdwan-applications/1708539371717015196/site/16190109915660160/details', + mos: seriesSummary({ values: [1.85, 4.4] }), + loss: seriesSummary({ values: [0.5] }), + jitter: seriesSummary({ values: [0] }), + }; + + const r = await wanAppRtpMosCheck.run(ctx); + assert.equal( + r.details.detailsUrl, + 'https://stratacloudmanager.paloaltonetworks.com/insights/operational/sdwan-applications/1708539371717015196/site/16190109915660160/details', + 'detailsUrl must be present in the check result so the renderer can emit the link', + ); + + // Missing URL on the container → check details have detailsUrl:null, + // so the renderer's conditional cleanly drops the link row. + delete ctx.sdwanData.appAudio.detailsUrl; + const r2 = await wanAppRtpMosCheck.run(ctx); + assert.equal(r2.details.detailsUrl, null, + 'missing container URL propagates as null (renderer no-ops on that)'); +}); + +test('wanAppRtp*: details.unit is the humanized display unit (not raw Prisma "percentage")', async () => { + // Regression for the "11.83percentage" rendering bug — the raw + // API unit string ("percentage", "milliseconds") is unreadable + // when concatenated to a value. The check-args unit ('%', 'ms') + // MUST override the raw unit in the details payload so the + // renderer displays "11.83%" not "11.83percentage". + const ctx = mkAppCtx({ + loss: seriesSummary({ values: [11.83], unit: 'percentage' }), + }); + const r = await wanAppRtpLossCheck.run(ctx); + assert.equal(r.details.unit, '%', 'display unit humanized in details'); + assert.equal(r.details.rawUnit, 'percentage', + 'raw Prisma unit retained for debugging (as rawUnit)'); +}); diff --git a/tests/voiceDiagRenderer.test.js b/tests/voiceDiagRenderer.test.js index 7d1cb02..f68d51c 100644 --- a/tests/voiceDiagRenderer.test.js +++ b/tests/voiceDiagRenderer.test.js @@ -309,7 +309,7 @@ test('renderer: WAN window banner hidden when no wan* check runs', () => { 'no need to advertise a WAN window when no WAN check ran'); }); -test('renderer: WAN window banner formats: 15m / 1h / 6h / 1d', () => { +test('renderer: WAN window banner formats: 15m / 1h / 6h / 1d / 7d', () => { const wanResult = [ R('wanLatency', 'ok', 'ok', null, { total: 1, ok: 1, warn: 0, error: 0, @@ -317,14 +317,139 @@ test('renderer: WAN window banner formats: 15m / 1h / 6h / 1d', () => { }), ]; const cases = [ - { min: 15, expect: 'WAN window: 15m' }, - { min: 60, expect: 'WAN window: 1h' }, - { min: 360, expect: 'WAN window: 6h' }, - { min: 1440, expect: 'WAN window: 1d' }, - { min: 45, expect: 'WAN window: 45m' }, + { min: 15, expect: 'WAN window: 15m' }, + { min: 60, expect: 'WAN window: 1h' }, + { min: 360, expect: 'WAN window: 6h' }, + { min: 1440, expect: 'WAN window: 1d' }, + { min: 10080, expect: 'WAN window: 7d' }, // new default + { min: 45, expect: 'WAN window: 45m' }, ]; for (const c of cases) { const md = renderVoiceDiagMarkdown(wanResult, { storeNum: '1', wanWindowMinutes: c.min }); assert.ok(md.includes(c.expect), `${c.min} minutes → "${c.expect}", got:\n${md}`); } }); + +// ─── Per-app audio details renderer ───────────────────────────────── +// +// renderAppAudioDetails is triggered by shape detection on +// {appName, worst, validSamples} — the fixture below carries all +// three so the shape-aware branch fires (rather than the JSON dump +// fallback which is what we're guarding against). appName is asserted +// verbatim from the fixture (not hardcoded to a specific app) — see +// wanAppRtpMos.js for the tenant-configurable contract. + +test('renderer: per-app audio MOS details render worst-window + threshold + range (Webex_Calling_RTP)', () => { + const wanResult = R( + 'wanAppRtpMos', + 'error', + 'Actual voice traffic (Webex_Calling_RTP) worst-window MOS: 1.60 < 3.5 (avg 4.27 over 33 valid samples)…', + null, + { + appName: 'Webex_Calling_RTP', + worst: 1.60, avg: 4.27, min: 1.60, max: 4.41, p95: 4.28, + samples: 55, validSamples: 33, interval: '5min', + warnThresh: 4.0, errorThresh: 3.5, + standardLabel: 'warn < 4, error < 3.5', + unit: '', + badSampleCount: 4, badSamplePct: 12, + }, + 'SD-WAN Voice Traffic MOS (worst window)', + ); + const md = renderVoiceDiagMarkdown([wanResult], { storeNum: '127', detailed: true }); + + // The app name is threaded from the fixture (appAudio.appName) — the + // renderer must not hardcode "rtp-base" so tenants using + // Webex_Calling_RTP, MS_Teams_RTP, etc. render correctly. + assert.match(md, /App: Webex_Calling_RTP \(voice traffic, DPI\)/); + assert.match(md, /Threshold: warn < 4, error < 3\.5/); + assert.match(md, /Worst window: \*\*1\.6\*\*/); + assert.match(md, /Avg: 4\.27/); + assert.match(md, /p95: 4\.28/); + assert.match(md, /Range: 1\.6 – 4\.41 across 33\/55 samples @ 5min/); + assert.match(md, /Time in warn\/error: 4 samples/); + // Absence check — the fallback JSON key:value dump must NOT appear. + assert.doesNotMatch(md, /"values":/, 'shape-aware formatter must not dump raw JSON'); +}); + +test('renderer: per-app audio details fall back to "voice" when appName absent (defensive)', () => { + // If a fixture / integration test forgets to set appName (or a + // future refactor drops it), the renderer must not crash or render + // "undefined" — fall back to the generic "voice" label. + const wanResult = R( + 'wanAppRtpMos', 'ok', 'ok', + null, + { + appName: null, + worst: 4.4, avg: 4.5, min: 4.4, max: 4.6, p95: 4.5, + samples: 12, validSamples: 12, interval: '5min', + warnThresh: 4.0, errorThresh: 3.5, + unit: '', badSampleCount: 0, badSamplePct: 0, + }, + 'SD-WAN Voice Traffic MOS (worst window)', + ); + const md = renderVoiceDiagMarkdown([wanResult], { storeNum: '127', detailed: true }); + assert.match(md, /App: voice \(voice traffic, DPI\)/, + 'null appName must fall back to generic "voice" label'); +}); + +test('renderer: per-app audio details renders "View in Prisma UI" deep link when detailsUrl is set', () => { + const wanResult = R( + 'wanAppRtpMos', 'error', 'msg', + null, + { + appName: 'Webex_Calling_RTP', + worst: 1.85, avg: 3.55, min: 1.85, max: 4.41, p95: 4.28, + samples: 288, validSamples: 288, interval: '5min', + warnThresh: 4.0, errorThresh: 3.5, + standardLabel: 'warn < 4, error < 3.5', + unit: '', + badSampleCount: 42, badSamplePct: 15, + detailsUrl: 'https://stratacloudmanager.paloaltonetworks.com/insights/operational/sdwan-applications/1708539371717015196/site/16190109915660160/details', + }, + 'SD-WAN Voice Traffic MOS (worst window)', + ); + const md = renderVoiceDiagMarkdown([wanResult], { storeNum: '127', detailed: true }); + assert.match( + md, + /\[View in Prisma UI\]\(https:\/\/stratacloudmanager\.paloaltonetworks\.com\/insights\/operational\/sdwan-applications\/1708539371717015196\/site\/16190109915660160\/details\)/, + 'deep link must render as a markdown link so Webex chats it as a clickable URL', + ); +}); + +test('renderer: per-app audio details omits Prisma link when detailsUrl is not set', () => { + const wanResult = R( + 'wanAppRtpMos', 'error', 'msg', + null, + { + appName: 'Webex_Calling_RTP', + worst: 1.85, avg: 3.55, min: 1.85, max: 4.41, p95: 4.28, + samples: 288, validSamples: 288, interval: '5min', + warnThresh: 4.0, errorThresh: 3.5, + unit: '', + badSampleCount: 42, badSamplePct: 15, + }, + 'SD-WAN Voice Traffic MOS (worst window)', + ); + const md = renderVoiceDiagMarkdown([wanResult], { storeNum: '127', detailed: true }); + assert.doesNotMatch(md, /View in Prisma UI/); +}); + +test('renderer: per-app audio details — clean window says "0 samples (clean throughout window)"', () => { + const wanResult = R( + 'wanAppRtpLoss', 'ok', 'ok', + null, + { + appName: 'Webex_Calling_RTP', + worst: 0, avg: 0, min: 0, max: 0, p95: 0, + samples: 288, validSamples: 288, interval: '5min', + warnThresh: 5, errorThresh: 15, + standardLabel: 'warn > 5%, error > 15%', + unit: '%', + badSampleCount: 0, badSamplePct: 0, + }, + 'SD-WAN Voice Traffic Packet Loss (worst window)', + ); + const md = renderVoiceDiagMarkdown([wanResult], { storeNum: '127', detailed: true }); + assert.match(md, /Time in warn\/error: 0 samples \(clean throughout window\)/); +});