Adds three new SD-WAN checks (wanAppRtpMos/Loss/Jitter) that measure
REAL voice-traffic quality on actual RTP frames via Prisma DPI, not
synthetic link probes. Graded against the WORST 5-minute window so
transient degradation the 24h link-probe averages smooth away
actually surfaces.
Voice-app selection is tenant-configurable via PRISMA_APP_ID_VOICE +
PRISMA_APP_NAME_VOICE (Webex_Calling_RTP recommended for Webex
Calling shops — the Webex-specific DPI signature excludes non-Webex
UDP noise). Legacy PRISMA_APP_ID_RTP_BASE still honored with a
one-time deprecation warning.
Widens the default WAN look-back from 24h to 7 days: per-app metrics
only get datapoints when calls actually happen, so sporadic Webex
Calling stores (3-4 calls/day) need a wider window for worst-window
statistics to be meaningful. Interval picker snaps 7d to 1hour
buckets (168 pts) to keep payloads bounded while preserving
worst-hour granularity. Hard-capped at 7d — beyond that Prisma
downsamples to 1-day buckets and the signal collapses.
Also:
- Client-side concurrency limiter (PRISMA_MAX_INFLIGHT, default 3)
to prevent 429 cascades when /voicediag fans out 10+ parallel
metric fetches
- "View in Prisma UI" deep links in both /phonestatus WAN follow-up
and /voicediag details, threading through a new
integrations/paloalto/urls.js builder
- humanizeMetricUnit maps raw API unit strings ("percentage",
"milliseconds") to display symbols ("%", "ms") to fix
"11.83percentage" leaking to the UI
- getAppAudio envelope distinguishes not-configured / fetch-failed /
no-traffic states so misleading "set env var" messages don't fire
when the real problem is a 429
Co-authored-by: Cursor <cursoragent@cursor.com>
471 lines
18 KiB
JavaScript
471 lines
18 KiB
JavaScript
// src/services/voiceDiag/checks/wan/_helpers.js
|
||
//
|
||
// Shared kill-switch + threshold accessors for the /voicediag WAN
|
||
// bucket. Mirrors services/voiceDiag/checks/port/_helpers.js in
|
||
// intent — one file per check for the actual logic, this file for
|
||
// the common env-reading and formatting boilerplate.
|
||
//
|
||
// Threshold accessors read `process.env` at *call time* rather than
|
||
// at import time. That's what lets tests do `process.env.X = '...'`
|
||
// + await the check without needing a module cache reset. The
|
||
// downside is a couple of extra env reads per run, which is nothing
|
||
// against the network work each WAN check does (or doesn't do — the
|
||
// checks reuse data already fetched during buildContext()).
|
||
|
||
/**
|
||
* Global kill-switch for the WAN bucket. Env
|
||
* `WAN_STANDARD_ENABLED=false` returns a short "WAN checks are
|
||
* currently disabled" skip result that every WAN check can return
|
||
* verbatim. Feature-config + port-hygiene checks are intentionally
|
||
* *not* gated by this — this only silences the SD-WAN bucket while
|
||
* the underlying Prisma tenant is being reconfigured or the
|
||
* integration is being validated.
|
||
*
|
||
* Same contract as port/_helpers.js:maybeSkippedByKillSwitch —
|
||
* returns null when enabled (caller proceeds) or a full CheckResult
|
||
* with `status: 'skipped'` when disabled (caller returns it as-is).
|
||
*
|
||
* @param {object} check the check descriptor (used for label only)
|
||
* @returns {null | {status:'skipped', message:string, details:object, remediation:null}}
|
||
*/
|
||
export function maybeSkippedByKillSwitch(check) {
|
||
const raw = String(process.env.WAN_STANDARD_ENABLED ?? 'true').toLowerCase().trim();
|
||
const enabled = !(raw === 'false' || raw === '0' || raw === 'no' || raw === 'off');
|
||
if (enabled) return null;
|
||
return {
|
||
status: 'skipped',
|
||
message:
|
||
`${check.label} skipped — WAN_STANDARD_ENABLED=false (SD-WAN checks are silenced).`,
|
||
details: { killSwitch: 'WAN_STANDARD_ENABLED', value: raw },
|
||
remediation: null,
|
||
};
|
||
}
|
||
|
||
// ─── Threshold accessors ──────────────────────────────────────────
|
||
//
|
||
// Defaults match the ITU-T G.114 / RFC 3550 references for voice
|
||
// quality. Any of these can be overridden per-tenant via env; the
|
||
// renderer + the checks read from the same accessors so the icon
|
||
// in the phonestatus follow-up always agrees with the verdict in
|
||
// voicediag.
|
||
|
||
function num(envKey, fallback) {
|
||
const raw = Number(process.env[envKey]);
|
||
return Number.isFinite(raw) ? raw : fallback;
|
||
}
|
||
|
||
// Latency (one-way estimate, milliseconds)
|
||
export const getLatencyWarnMs = () => num('WAN_STANDARD_LATENCY_WARN_MS', 150);
|
||
export const getLatencyErrorMs = () => num('WAN_STANDARD_LATENCY_ERROR_MS', 400);
|
||
|
||
// Jitter (inter-packet arrival variation, milliseconds)
|
||
export const getJitterWarnMs = () => num('WAN_STANDARD_JITTER_WARN_MS', 30);
|
||
export const getJitterErrorMs = () => num('WAN_STANDARD_JITTER_ERROR_MS', 50);
|
||
|
||
// Packet loss (percent 0-100)
|
||
export const getLossWarnPct = () => num('WAN_STANDARD_LOSS_WARN_PCT', 1);
|
||
export const getLossErrorPct = () => num('WAN_STANDARD_LOSS_ERROR_PCT', 3);
|
||
|
||
// MOS (Mean Opinion Score, 1.0-5.0 — HIGHER is better, so warn/err
|
||
// mean "value FALLS BELOW this")
|
||
export const getMosWarn = () => num('WAN_STANDARD_MOS_WARN', 4.0);
|
||
export const getMosError = () => num('WAN_STANDARD_MOS_ERROR', 3.5);
|
||
|
||
// Healthscore (Prisma AIOps roll-up, 0-100 — HIGHER is better)
|
||
export const getHealthscoreWarn = () => num('WAN_STANDARD_HEALTHSCORE_WARN', 80);
|
||
export const getHealthscoreError = () => num('WAN_STANDARD_HEALTHSCORE_ERROR', 60);
|
||
|
||
// ─── Per-application (DPI) audio-quality thresholds ──────────────
|
||
// Applied against the WORST-window value in the series (not the avg)
|
||
// since these signals come as 288-point 24h time series and the whole
|
||
// point of app-quality checks is to catch transient degradation the
|
||
// LQM link-probe average smooths away. Defaults match the ITU-T
|
||
// references but are DELIBERATELY less strict than the link-probe
|
||
// thresholds because false positives from a single bad 5-min window
|
||
// would drown the operator in noise. Tune via env if your traffic
|
||
// mix is more sensitive.
|
||
//
|
||
// MOS (worst-window, lower is worse — call quality drops)
|
||
export const getAppMosWarn = () => num('WAN_STANDARD_APP_MOS_WARN', 4.0);
|
||
export const getAppMosError = () => num('WAN_STANDARD_APP_MOS_ERROR', 3.5);
|
||
|
||
// Loss (worst-window percentage, higher is worse)
|
||
export const getAppLossWarnPct = () => num('WAN_STANDARD_APP_LOSS_WARN_PCT', 5);
|
||
export const getAppLossErrorPct = () => num('WAN_STANDARD_APP_LOSS_ERROR_PCT', 15);
|
||
|
||
// Jitter (worst-window ms, higher is worse)
|
||
export const getAppJitterWarnMs = () => num('WAN_STANDARD_APP_JITTER_WARN_MS', 30);
|
||
export const getAppJitterErrorMs = () => num('WAN_STANDARD_APP_JITTER_ERROR_MS', 50);
|
||
|
||
// ─── Utilities used by multiple checks ────────────────────────────
|
||
|
||
/**
|
||
* Safe number-of-links accessor. WAN checks that grade the whole
|
||
* site (healthscore, alarms) don't care about link count, but the
|
||
* per-link checks below use it to short-circuit when there are no
|
||
* links at all (returns skipped with an actionable message).
|
||
*/
|
||
export function getLinks(ctx) {
|
||
const raw = ctx?.sdwanData?.links;
|
||
return Array.isArray(raw) ? raw : [];
|
||
}
|
||
|
||
/** Label a link for messages. Kept short so aggregate messages
|
||
* don't blow past Webex's chat readability. */
|
||
export function labelForLink(link) {
|
||
const name = link?.interfaceName || link?.interfaceId || 'link';
|
||
const el = link?.elementName ? `@${link.elementName}` : '';
|
||
const tx = link?.transportType ? ` [${link.transportType}]` : '';
|
||
return `${name}${el}${tx}`;
|
||
}
|
||
|
||
/**
|
||
* Accessor for the per-app audio-quality summary computed in
|
||
* sdwanEnrichment.js. Returns a small envelope so the caller can
|
||
* distinguish four states cleanly:
|
||
*
|
||
* {configured:false, summary:null, fetchError:null}
|
||
* Voice-app env not set → checks return "not configured"
|
||
* skipped. See sdwanEnrichment.js:resolveVoiceAppConfig() for
|
||
* the env-var contract (PRISMA_APP_ID_VOICE, or
|
||
* PRISMA_APP_ID_RTP_BASE for backwards compat). This is a
|
||
* config decision, not a transient failure.
|
||
*
|
||
* {configured:true, summary:null, fetchError:'<msg>'}
|
||
* Env set + Prisma fetch failed (429 storm, timeout, schema
|
||
* drift). Different UX from "not configured" — surfaces the
|
||
* real error so the operator can retry or fix.
|
||
*
|
||
* {configured:true, summary:{...},fetchError:null}
|
||
* Fetch succeeded but Prisma returned zero valid samples
|
||
* (no matching RTP traffic in the window). Skipped with a
|
||
* "no traffic seen" message.
|
||
*
|
||
* {configured:true, summary:{...validSamples>0...}, fetchError:null}
|
||
* Green path — check grades against the summary.
|
||
*
|
||
* @param {object} ctx
|
||
* @param {'mos'|'loss'|'jitter'|'bandwidth'} which
|
||
*/
|
||
export function getAppAudio(ctx, which) {
|
||
const container = ctx?.sdwanData?.appAudio || null;
|
||
if (!container) {
|
||
return {
|
||
configured: false,
|
||
summary: null,
|
||
fetchError: null,
|
||
appName: null,
|
||
detailsUrl: null,
|
||
};
|
||
}
|
||
// Error-scope names are app-agnostic ("app.voice.<metric>") so the
|
||
// check doesn't have to know which specific voice app is
|
||
// configured — it just asks "was the fetch for MOS on the voice
|
||
// app OK?" See sdwanEnrichment.js:recordFailure() for the writer.
|
||
const scope = `app.voice.${which}`;
|
||
const errRec = Array.isArray(ctx?.sdwanData?.errors)
|
||
? ctx.sdwanData.errors.find((e) => e?.scope === scope)
|
||
: null;
|
||
return {
|
||
configured: true,
|
||
summary: container[which] || null,
|
||
fetchError: errRec?.message || null,
|
||
appName: container.appName || 'voice',
|
||
// SCM UI deep-link computed once in sdwanEnrichment.js. Threaded
|
||
// through here so the check details carry it into the voiceDiag
|
||
// renderer as a "View in Prisma UI" link.
|
||
detailsUrl: container.detailsUrl || null,
|
||
};
|
||
}
|
||
|
||
/**
|
||
* Humanize the raw Prisma unit string into a display-friendly suffix.
|
||
* The API returns strings like `"percentage"` and `"milliseconds"`
|
||
* which read like error messages when concatenated into a UI value
|
||
* (`"11.83percentage"`). This normalizes to the units an operator
|
||
* actually recognizes.
|
||
*
|
||
* Exposed so both the renderer and the check message use the same
|
||
* mapping — divergence there would produce mismatched icons vs
|
||
* numbers.
|
||
*/
|
||
export function humanizeMetricUnit(raw) {
|
||
if (!raw) return '';
|
||
const lc = String(raw).toLowerCase();
|
||
if (lc === 'percentage' || lc === 'percent') return '%';
|
||
if (lc === 'milliseconds' || lc === 'ms') return 'ms';
|
||
if (lc === 'count' || lc === 'gauge' || lc === 'score') return '';
|
||
if (lc === 'kbps') return 'kbps';
|
||
if (lc === 'mbps') return 'Mbps';
|
||
if (lc === 'bps') return 'bps';
|
||
if (lc === 'seconds' || lc === 's') return 's';
|
||
// Anything else — return as-is, prefixed with a space so it's
|
||
// visually separated from the number (`"12 something"` reads
|
||
// better than `"12something"`).
|
||
return ` ${raw}`;
|
||
}
|
||
|
||
/**
|
||
* Grade a per-app audio quality summary against warn/error thresholds.
|
||
* Handles the four cases the envelope from `getAppAudio()` can be in:
|
||
* not-configured, fetch-failed, no-traffic, gradeable.
|
||
*
|
||
* @param {object} args
|
||
* @param {object} args.audio envelope from getAppAudio():
|
||
* {configured, summary, fetchError, appName}
|
||
* @param {string} args.label display label ("audio MOS", "audio loss", …)
|
||
* @param {string} args.unit display unit ("%", "ms", "")
|
||
* @param {number} args.warnThresh
|
||
* @param {number} args.errorThresh
|
||
* @param {boolean} args.lowIsBad true → warn/error when the
|
||
* WORST-window VALUE FALLS BELOW
|
||
* the threshold (MOS). false →
|
||
* warn/error when the WORST-window
|
||
* value RISES ABOVE (loss, jitter).
|
||
* @param {string} [args.standardLabel] optional inline "warn > X"
|
||
* description embedded in the message
|
||
* @returns {object} CheckResult
|
||
*/
|
||
export function evaluateAppAudioMetric({
|
||
audio, label, unit, warnThresh, errorThresh, lowIsBad = false,
|
||
standardLabel,
|
||
}) {
|
||
const appName = audio?.appName || 'the voice app';
|
||
|
||
if (!audio || audio.configured === false) {
|
||
return {
|
||
status: 'skipped',
|
||
message:
|
||
`Per-app ${label} unavailable — set PRISMA_APP_ID_VOICE in the ` +
|
||
`environment (with optional PRISMA_APP_NAME_VOICE for the display ` +
|
||
`name) to enable per-application DPI metrics.`,
|
||
details: null,
|
||
remediation: null,
|
||
};
|
||
}
|
||
|
||
if (audio.fetchError) {
|
||
return {
|
||
status: 'skipped',
|
||
message:
|
||
`Per-app ${label} for ${appName}: Prisma fetch failed — ${audio.fetchError}. ` +
|
||
`Common cause: parallel-request rate limiting (429). Retry in ~30 seconds.`,
|
||
details: { fetchError: audio.fetchError, appName },
|
||
remediation: null,
|
||
};
|
||
}
|
||
|
||
const summary = audio.summary;
|
||
if (!summary) {
|
||
return {
|
||
status: 'skipped',
|
||
message:
|
||
`Per-app ${label} for ${appName}: no summary produced (Prisma returned an ` +
|
||
`empty or malformed response). Check paloalto:metrics logs for the raw payload.`,
|
||
details: null,
|
||
remediation: null,
|
||
};
|
||
}
|
||
if (!summary.validSamples || summary.validSamples === 0) {
|
||
return {
|
||
status: 'skipped',
|
||
message:
|
||
`Per-app ${label} for ${appName}: Prisma returned ` +
|
||
`${summary.samples || 0} datapoint(s), all null — no voice traffic ` +
|
||
`matched by DPI in the window.`,
|
||
details: summary,
|
||
remediation: null,
|
||
};
|
||
}
|
||
|
||
const worst = lowIsBad ? summary.min : summary.max;
|
||
const grade = (v) => {
|
||
if (v == null) return 'skipped';
|
||
if (lowIsBad) {
|
||
if (v < errorThresh) return 'error';
|
||
if (v < warnThresh) return 'warn';
|
||
return 'ok';
|
||
}
|
||
if (v > errorThresh) return 'error';
|
||
if (v > warnThresh) return 'warn';
|
||
return 'ok';
|
||
};
|
||
|
||
const status = grade(worst);
|
||
|
||
// Percent of samples that were in warn or error range — the "how
|
||
// often was it bad" number that turns the worst-window verdict into
|
||
// an actionable "N% of your voice calls saw this" statistic.
|
||
const bad = summary.values.filter(
|
||
(v) => grade(v) === 'warn' || grade(v) === 'error',
|
||
).length;
|
||
const badPct = summary.validSamples > 0
|
||
? Math.round((bad / summary.validSamples) * 100)
|
||
: 0;
|
||
|
||
// Store both the raw API unit (for debugging) and the humanized
|
||
// display unit (what the renderer + message use). Overriding the
|
||
// spread unit with the check-arg unit lets each check pick a nice
|
||
// suffix ('%', 'ms') without depending on the API's verbose one.
|
||
const details = {
|
||
...summary,
|
||
rawUnit: summary.unit || null,
|
||
unit,
|
||
warnThresh,
|
||
errorThresh,
|
||
standardLabel: standardLabel || null,
|
||
worst,
|
||
badSampleCount: bad,
|
||
badSamplePct: badPct,
|
||
appName,
|
||
// SCM UI deep-link. The renderer surfaces this as a
|
||
// "View in Prisma UI" markdown link so an operator can jump from
|
||
// the diag output straight to the Prisma UI dashboard for the
|
||
// exact site+app combination without hunting for it.
|
||
detailsUrl: audio.detailsUrl || null,
|
||
};
|
||
|
||
const worstStr = worst == null ? '—' : `${worst}${unit}`;
|
||
const avgStr = summary.avg == null ? '—' : `${summary.avg}${unit}`;
|
||
const cmp = lowIsBad ? '<' : '>';
|
||
const threshStr = status === 'error'
|
||
? `${cmp} ${errorThresh}${unit}`
|
||
: `${cmp} ${warnThresh}${unit}`;
|
||
|
||
if (status === 'error' || status === 'warn') {
|
||
return {
|
||
status,
|
||
message:
|
||
`Actual voice traffic (${appName}) worst-window ${label}: ${worstStr} ${threshStr} ` +
|
||
`(avg ${avgStr} over ${summary.validSamples} × ${summary.interval || '5min'} samples). ` +
|
||
`${badPct}% of samples were ${status === 'error' ? 'in the error range' : 'past warn'}. ` +
|
||
`This is measured on real RTP traffic — the WAN link probes may still look fine.`,
|
||
details,
|
||
remediation: null,
|
||
};
|
||
}
|
||
|
||
return {
|
||
status: 'ok',
|
||
message:
|
||
`Actual voice traffic (${appName}) ${label} within range: worst ${worstStr}, ` +
|
||
`avg ${avgStr} across ${summary.validSamples} × ${summary.interval || '5min'} samples.`,
|
||
details,
|
||
remediation: null,
|
||
};
|
||
}
|
||
|
||
/**
|
||
* Evaluate a numeric per-link metric against warn/error thresholds
|
||
* across every path, returning a CheckResult. Worst path drives
|
||
* severity — a single bad link takes precedence over three good
|
||
* ones because "the phone routed over the bad path sounds terrible"
|
||
* is the ticket we're trying to catch.
|
||
*
|
||
* Consolidated here so the four LQM checks (latency / jitter /
|
||
* loss / mos) don't duplicate 40 lines of comparison logic — they
|
||
* pass config in, get a CheckResult out.
|
||
*
|
||
* @param {object} args
|
||
* @param {Array<object>} args.links ctx.sdwanData.links
|
||
* @param {string} args.metricKey row property to read (e.g. 'latencyMs')
|
||
* @param {string} args.label label for messages (e.g. 'latency')
|
||
* @param {string} args.unit display unit (e.g. 'ms', '%', '')
|
||
* @param {number} args.warnThresh
|
||
* @param {number} args.errorThresh
|
||
* @param {boolean} args.lowIsBad true → warn/error when value FALLS BELOW threshold (MOS)
|
||
* @param {string} args.standardLabel for the "compliant range" phrasing
|
||
*/
|
||
export function evaluatePerLinkMetric({
|
||
links,
|
||
metricKey,
|
||
label,
|
||
unit,
|
||
warnThresh,
|
||
errorThresh,
|
||
lowIsBad = false,
|
||
standardLabel,
|
||
}) {
|
||
if (!Array.isArray(links) || links.length === 0) {
|
||
return {
|
||
status: 'skipped',
|
||
message: `No WAN paths reported for this site.`,
|
||
details: null,
|
||
remediation: null,
|
||
};
|
||
}
|
||
|
||
const withValue = links.filter((l) => Number.isFinite(l[metricKey]));
|
||
if (withValue.length === 0) {
|
||
return {
|
||
status: 'skipped',
|
||
message: `${label} not available from Prisma for any path (partial fetch).`,
|
||
details: null,
|
||
remediation: null,
|
||
};
|
||
}
|
||
|
||
const grade = (v) => {
|
||
if (lowIsBad) {
|
||
if (v < errorThresh) return 'error';
|
||
if (v < warnThresh) return 'warn';
|
||
return 'ok';
|
||
}
|
||
if (v > errorThresh) return 'error';
|
||
if (v > warnThresh) return 'warn';
|
||
return 'ok';
|
||
};
|
||
|
||
const perLink = withValue.map((l) => ({
|
||
link: labelForLink(l),
|
||
value: l[metricKey],
|
||
interfaceId: l.interfaceId,
|
||
verdict: grade(l[metricKey]),
|
||
}));
|
||
|
||
const errors = perLink.filter((r) => r.verdict === 'error');
|
||
const warns = perLink.filter((r) => r.verdict === 'warn');
|
||
|
||
const worstLine = (rows) =>
|
||
rows.map((r) => `${r.link} (${r.value}${unit})`).join(', ');
|
||
|
||
const details = {
|
||
total: perLink.length,
|
||
ok: perLink.filter((r) => r.verdict === 'ok').length,
|
||
warn: warns.length,
|
||
error: errors.length,
|
||
perLink,
|
||
warnThresh,
|
||
errorThresh,
|
||
standardLabel,
|
||
};
|
||
|
||
if (errors.length > 0) {
|
||
return {
|
||
status: 'error',
|
||
message:
|
||
`${errors.length} path(s) with ${label} past error threshold ` +
|
||
`(${standardLabel}): ${worstLine(errors)}.`,
|
||
details,
|
||
remediation: null,
|
||
};
|
||
}
|
||
|
||
if (warns.length > 0) {
|
||
return {
|
||
status: 'warn',
|
||
message:
|
||
`${warns.length} path(s) with elevated ${label} ` +
|
||
`(${standardLabel}): ${worstLine(warns)}.`,
|
||
details,
|
||
remediation: null,
|
||
};
|
||
}
|
||
|
||
return {
|
||
status: 'ok',
|
||
message: `All ${perLink.length} path(s) with ${label} in the compliant range (${standardLabel}).`,
|
||
details,
|
||
remediation: null,
|
||
};
|
||
}
|