jmcqueen
bff9e8dd0b
The Cisco Webex SDK's Mercury WebSocket (used by the bot in WebSocket
mode) can die silently — network blip, WDM device TTL expiring, Cisco-
side hiccup — without any error the framework surfaces. When this
happens, the HTTP paths (webhook → Webex message API) keep working but
the bot silently stops receiving commands. This is the classic
"webhook alerts still arrive but the bot ignores me" failure mode.
Detect it by polling the SDK's live `webex.internal.mercury.connected`
boolean every 60s. Track consecutive misses:
- After ~2 min disconnected: log a warning
- After ~5 min disconnected: log an error and trigger graceful
shutdown, so Docker's `restart: unless-stopped` policy brings us
back with a fresh Mercury socket
Complementary changes:
- /health now returns 503 when the watchdog considers the bot dead,
with `bot.mercuryConnected`, `consecutiveFailures`, `lastHealthyAt`
in the JSON body. Docker HEALTHCHECK will start failing too, which
helps external autoheal / K8s liveness probes catch it before the
in-process exit fires. During SMOKE_TEST=true the bot state is
reported as "skipped-smoke-test" so smoke tests still pass.
- Framework 'log' events are forwarded into our structured logger so
framework-internal diagnostics (device registration issues,
membership rule denials, etc.) are actually visible in logs.
- `removeDeviceRegistrationsOnStart: true` cleans up WDM device
registrations left behind by previous silently-dead instances so
they don't accumulate over time. Safe for single-instance
deployments; comment flags the multi-instance caveat.
README updated to document the new health semantics and watchdog.
Co-authored-by: Cursor <cursoragent@cursor.com>
1006 lines
41 KiB
JavaScript
1006 lines
41 KiB
JavaScript
require('dotenv').config();
|
||
const express = require('express');
|
||
const axios = require('axios');
|
||
const Framework = require('webex-node-bot-framework');
|
||
|
||
const app = express();
|
||
const PORT = process.env.PORT || 3000;
|
||
|
||
// ======================
|
||
// STARTUP VALIDATION (fail fast on missing critical config)
|
||
// ======================
|
||
const REQUIRED_ENV = [
|
||
'WEBEX_BOT_TOKEN',
|
||
'WEBEX_ROOM_ID',
|
||
'APPSPACE_INSTANCE_URL',
|
||
'APPSPACE_SUBJECT_ID',
|
||
'APPSPACE_REFRESH_TOKEN',
|
||
'APPSPACE_API_BASE_URL'
|
||
];
|
||
|
||
const missing = REQUIRED_ENV.filter(k => !process.env[k]);
|
||
if (missing.length > 0) {
|
||
console.error('❌ Missing required environment variables:');
|
||
missing.forEach(k => console.error(` - ${k}`));
|
||
console.error(' See .env.example for details.');
|
||
process.exit(1);
|
||
}
|
||
|
||
// Logging configuration for Docker / production friendliness
|
||
const isVerbose = process.env.DEBUG === 'true' || process.env.NODE_ENV !== 'production';
|
||
const useJsonLogs = process.env.LOG_FORMAT === 'json' || process.env.NODE_ENV === 'production';
|
||
|
||
/**
|
||
* Minimal structured logger.
|
||
* - Human readable (with emojis) in dev
|
||
* - JSON lines when LOG_FORMAT=json or NODE_ENV=production (great for Docker log collectors)
|
||
*/
|
||
const logger = {
|
||
info: (msg, meta = {}) => log('info', msg, meta),
|
||
warn: (msg, meta = {}) => log('warn', msg, meta),
|
||
error: (msg, meta = {}) => log('error', msg, meta),
|
||
debug: (msg, meta = {}) => { if (isVerbose) log('debug', msg, meta); }
|
||
};
|
||
|
||
// Normalize whatever was passed as `meta` into a plain object suitable for JSON
|
||
// logging. Errors have no enumerable own properties, so JSON.stringify(err) ⇒ "{}".
|
||
// We pull out the useful bits (name/message/stack + axios response shape) so failures
|
||
// are actually visible in the log stream.
|
||
function normalizeMeta(meta) {
|
||
if (meta == null) return {};
|
||
if (meta instanceof Error) {
|
||
const out = { error: meta.message, errorName: meta.name, stack: meta.stack };
|
||
if (meta.code) out.code = meta.code;
|
||
if (meta.response) {
|
||
out.responseStatus = meta.response.status;
|
||
out.responseData = meta.response.data;
|
||
}
|
||
return out;
|
||
}
|
||
if (typeof meta !== 'object') return { value: meta };
|
||
return meta;
|
||
}
|
||
|
||
function log(level, msg, meta = {}) {
|
||
const timestamp = new Date().toISOString();
|
||
const normMeta = normalizeMeta(meta);
|
||
if (useJsonLogs) {
|
||
const entry = { timestamp, level, msg, ...normMeta };
|
||
// Remove undefined
|
||
Object.keys(entry).forEach(k => entry[k] === undefined && delete entry[k]);
|
||
console.log(JSON.stringify(entry));
|
||
} else {
|
||
const emoji = level === 'error' ? '❌' : level === 'warn' ? '⚠️' : level === 'debug' ? '🐛' : 'ℹ️';
|
||
const metaStr = Object.keys(normMeta).length ? ' ' + JSON.stringify(normMeta) : '';
|
||
const out = level === 'error' ? console.error : console.log;
|
||
out(`${emoji} ${msg}${metaStr}`);
|
||
}
|
||
}
|
||
|
||
// Will be assigned when the server starts listening
|
||
let server;
|
||
let framework;
|
||
|
||
// ======================
|
||
// BOT LIVENESS STATE (populated by the Mercury watchdog below)
|
||
// ======================
|
||
// Represents whether the Webex bot's WebSocket (Mercury) transport is currently
|
||
// healthy. Read by /health so container orchestrators can observe the state.
|
||
// Set to `true` initially so we don't fail the healthcheck during the startup
|
||
// window before the framework has finished initializing.
|
||
const botHealth = {
|
||
mercuryConnected: null, // last-observed value of webex.internal.mercury.connected (null = not-yet-checked)
|
||
consecutiveFailures: 0, // consecutive watchdog checks that saw a dead socket
|
||
lastCheckAt: null, // ISO timestamp of last watchdog check
|
||
lastHealthyAt: null, // ISO timestamp of last healthy check (for staleness reporting)
|
||
frameworkInitialized: false, // set true on framework 'initialized' event
|
||
exitingBecauseDead: false // set true when the watchdog is about to exit the process
|
||
};
|
||
|
||
// Graceful shutdown function (used by signals and crash handlers)
|
||
function shutdown(force = false) {
|
||
logger.info('🛑 Graceful shutdown initiated...');
|
||
|
||
const exit = (code = 0) => {
|
||
logger.info(`👋 Process exiting with code ${code}`);
|
||
process.exit(code);
|
||
};
|
||
|
||
// Stop the Webex framework (important for WebSocket mode to clean up device registration)
|
||
if (typeof framework !== 'undefined' && framework.stop) {
|
||
framework.stop().then(() => {
|
||
logger.info('✅ Webex framework stopped');
|
||
if (server) {
|
||
server.close(() => {
|
||
logger.info('✅ HTTP server closed');
|
||
exit(0);
|
||
});
|
||
} else {
|
||
exit(0);
|
||
}
|
||
}).catch((err) => {
|
||
logger.error('Error stopping framework:', err);
|
||
if (server) server.close(() => exit(1));
|
||
else exit(1);
|
||
});
|
||
} else if (server) {
|
||
server.close(() => {
|
||
logger.info('✅ HTTP server closed');
|
||
exit(0);
|
||
});
|
||
} else {
|
||
exit(0);
|
||
}
|
||
|
||
// ALWAYS arm a safety timeout. If any step above hangs (e.g. framework.stop()
|
||
// never resolves), we still need to exit before Docker's 10s SIGKILL. When called
|
||
// from a crash handler (`force=true`) the process is already unhealthy, so exit
|
||
// faster to minimize the window where we're wedged.
|
||
setTimeout(() => {
|
||
logger.error('⏱️ Graceful shutdown timed out. Forcing exit.');
|
||
exit(1);
|
||
}, force ? 3000 : 8000).unref();
|
||
}
|
||
|
||
// Basic crash handlers for container environments (Docker/K8s will restart on non-zero exit)
|
||
// Uncaught is fatal -> shutdown.
|
||
// UnhandledRejection (e.g. from Webex framework with bad creds or transient issues) just log; don't kill the main server.
|
||
process.on('uncaughtException', (err) => {
|
||
logger.error('Uncaught Exception. Initiating shutdown...', err);
|
||
shutdown(true);
|
||
});
|
||
|
||
process.on('unhandledRejection', (reason) => {
|
||
// `reason` is most often an Error but can be anything; the logger normalizes it.
|
||
// If something rejected with a non-Error value, wrap it so we still get a stack-y view.
|
||
const payload = reason instanceof Error
|
||
? reason
|
||
: { reason: typeof reason === 'object' ? JSON.stringify(reason) : String(reason) };
|
||
logger.error('Unhandled Rejection (logged, continuing)...', payload);
|
||
// Do not call shutdown - keep the HTTP server running (e.g. Webex bot errors shouldn't kill webhook path)
|
||
});
|
||
|
||
app.use(express.json());
|
||
|
||
// ======================
|
||
// APPSPACE TOKEN MANAGEMENT (with cooldown + longer timeout)
|
||
// ======================
|
||
let currentAccessToken = null;
|
||
let tokenExpiresAt = 0;
|
||
let lastRefreshAttempt = 0;
|
||
let inFlightAppspaceTokenPromise = null;
|
||
|
||
function invalidateAppspaceToken() {
|
||
currentAccessToken = null;
|
||
tokenExpiresAt = 0;
|
||
}
|
||
|
||
async function getValidAccessToken() {
|
||
const now = Math.floor(Date.now() / 1000);
|
||
|
||
// Use cached token if still valid (2-minute safety buffer)
|
||
if (currentAccessToken && now < tokenExpiresAt - 120) {
|
||
return currentAccessToken;
|
||
}
|
||
|
||
// Coalesce concurrent callers (e.g. burst webhooks / bot commands at startup)
|
||
// so we never fire more than one refresh in parallel.
|
||
if (inFlightAppspaceTokenPromise) return inFlightAppspaceTokenPromise;
|
||
|
||
// Prevent hammering the endpoint (minimum 10 seconds between refresh attempts)
|
||
if (now - lastRefreshAttempt < 10) {
|
||
logger.debug('Token refresh attempted too recently — using last known token if available');
|
||
if (currentAccessToken) return currentAccessToken;
|
||
}
|
||
|
||
lastRefreshAttempt = now;
|
||
|
||
const instanceUrl = process.env.APPSPACE_INSTANCE_URL;
|
||
if (!instanceUrl) {
|
||
throw new Error('APPSPACE_INSTANCE_URL is not set in .env');
|
||
}
|
||
|
||
inFlightAppspaceTokenPromise = (async () => {
|
||
logger.info('Refreshing Appspace access token...');
|
||
try {
|
||
const response = await axios.post(`${instanceUrl}/api/v3/authorization/token`, {
|
||
subjectType: "Application",
|
||
subjectId: process.env.APPSPACE_SUBJECT_ID,
|
||
grantType: "refreshToken",
|
||
refreshToken: process.env.APPSPACE_REFRESH_TOKEN
|
||
}, {
|
||
headers: { 'Content-Type': 'application/json' },
|
||
timeout: 30000 // Increased to 30 seconds
|
||
});
|
||
|
||
const data = response.data;
|
||
currentAccessToken = data.accessToken;
|
||
tokenExpiresAt = Math.floor(Date.now() / 1000) + (data.expiresIn || 3600);
|
||
|
||
logger.info('Appspace access token refreshed successfully', { expiresIn: data.expiresIn || 3600 });
|
||
|
||
return currentAccessToken;
|
||
} catch (err) {
|
||
logger.error('Token refresh failed', { error: err.response?.data || err.message });
|
||
const status = err.response?.status;
|
||
if (status === 401 || status === 403) {
|
||
invalidateAppspaceToken();
|
||
}
|
||
if (err.code === 'ECONNABORTED') {
|
||
logger.warn('Request timed out. Appspace token endpoint may be rate-limited.');
|
||
}
|
||
throw new Error('Could not obtain valid Appspace access token. Try again in 30-60 seconds.');
|
||
}
|
||
})().finally(() => {
|
||
inFlightAppspaceTokenPromise = null;
|
||
});
|
||
|
||
return inFlightAppspaceTokenPromise;
|
||
}
|
||
|
||
// ======================
|
||
// UTILITIES
|
||
// ======================
|
||
/**
|
||
* Normalize and format MDM timestamps (LastSystemSampleTime, LastSeen, etc.)
|
||
* Handles missing 'Z' suffix from some WS1/Appspace responses and formats in
|
||
* Eastern Time (America/New_York, DST-aware — so it renders as EST or EDT
|
||
* automatically depending on the date).
|
||
*/
|
||
function formatMdmTimestamp(ts) {
|
||
if (!ts) return 'Unknown';
|
||
let timestamp = ts.toString().trim();
|
||
// Some backends omit Z on what is effectively UTC; append if it looks like it needs it
|
||
if (!timestamp.endsWith('Z') && !timestamp.includes('+') && timestamp.includes('-')) {
|
||
timestamp += 'Z';
|
||
}
|
||
const date = new Date(timestamp);
|
||
if (isNaN(date.getTime())) {
|
||
return timestamp;
|
||
}
|
||
return date.toLocaleString('en-US', {
|
||
timeZone: 'America/New_York',
|
||
month: 'numeric',
|
||
day: 'numeric',
|
||
year: 'numeric',
|
||
hour: 'numeric',
|
||
minute: '2-digit',
|
||
hour12: true
|
||
});
|
||
}
|
||
|
||
/**
|
||
* Extract a consistent set of MDM facts + Workspace ONE console link from a device record.
|
||
* Handles the quirky casing and nested Id.Value shape in WS1 responses.
|
||
* IMPORTANT: Use WS1_CONSOLE_BASE_URL (not WS1_BASE_URL) for the link base, as the API hostname (as....awmdm.com)
|
||
* is different from the console hostname (cn....awmdm.com) and will produce broken links.
|
||
*/
|
||
function buildMdmFactsAndLink(mdmDevice) {
|
||
if (!mdmDevice) {
|
||
return { mdmFacts: [], mdmConsoleLink: null };
|
||
}
|
||
|
||
const model = mdmDevice.Model || mdmDevice.model || 'Unknown';
|
||
const os = mdmDevice.OperatingSystem || 'Unknown';
|
||
const compliance = mdmDevice.ComplianceStatus || mdmDevice.complianceStatus || 'Unknown';
|
||
|
||
const lastSampleTime = formatMdmTimestamp(mdmDevice.LastSystemSampleTime);
|
||
const lastSeenDisplay = formatMdmTimestamp(mdmDevice.LastSeen);
|
||
|
||
const deviceId = mdmDevice.Id?.Value || mdmDevice.Uuid || mdmDevice.id || '';
|
||
let mdmConsoleLink = null;
|
||
if (deviceId) {
|
||
// Always use the console hostname (e.g. cn1896.awmdm.com), not the API hostname (e.g. as1896.awmdm.com)
|
||
// WS1_BASE_URL is the API server; WS1_CONSOLE_BASE_URL (or equivalent) is for the web UI links.
|
||
const ws1Base = process.env.WS1_CONSOLE_BASE_URL || 'https://cn1896.awmdm.com';
|
||
mdmConsoleLink = `${ws1Base}/AirWatch/#/AirWatch/Device/Details/Summary/${deviceId}`;
|
||
}
|
||
|
||
const mdmFacts = [];
|
||
if (model && model !== 'Unknown') mdmFacts.push({ "title": "Model", "value": model });
|
||
if (os && os !== 'Unknown') mdmFacts.push({ "title": "OS Version", "value": os });
|
||
if (compliance && compliance !== 'Unknown') mdmFacts.push({ "title": "Compliance", "value": compliance });
|
||
if (lastSampleTime && lastSampleTime !== 'Unknown') mdmFacts.push({ "title": "Last Sample (ET)", "value": lastSampleTime });
|
||
if (lastSeenDisplay && lastSeenDisplay !== 'Unknown') mdmFacts.push({ "title": "Last Seen (ET)", "value": lastSeenDisplay });
|
||
|
||
return { mdmFacts, mdmConsoleLink };
|
||
}
|
||
|
||
/**
|
||
* Construct the complete Adaptive Card payload for Webex.
|
||
* Keeps presentation logic out of the request handler.
|
||
*
|
||
* Note: Webex currently supports a maximum of Adaptive Cards 1.3
|
||
* (1.4+ is in the engineering backlog with no ETA as of 2025/2026).
|
||
* See: https://developer.webex.com/docs/buttons-and-cards
|
||
* Schema explorer: https://adaptivecards.io/explorer/
|
||
*/
|
||
function buildDeviceAlertCard({ eventKey, data, accentColor, emoji, appspaceLink, mdmFacts = [], mdmConsoleLink = null }) {
|
||
const adaptiveCard = {
|
||
"$schema": "http://adaptivecards.io/schemas/adaptive-card.json",
|
||
"type": "AdaptiveCard",
|
||
"version": "1.3", // Webex currently supports a maximum of Adaptive Cards 1.3 (1.4+ is in backlog with no ETA)
|
||
"body": [
|
||
{
|
||
"type": "Container",
|
||
"style": accentColor,
|
||
"bleed": true,
|
||
"items": [
|
||
{ "type": "TextBlock", "text": `${emoji} Appspace Device Alert`, "weight": "bolder", "size": "medium", "wrap": true }
|
||
]
|
||
},
|
||
{
|
||
"type": "FactSet",
|
||
"facts": [
|
||
{ "title": "Event", "value": eventKey },
|
||
{ "title": "Device", "value": data.deviceName || 'Unknown' },
|
||
{ "title": "Location", "value": data.locationName || 'Unknown' },
|
||
{ "title": "Type", "value": data.deviceType || 'N/A' },
|
||
{ "title": "IP", "value": data.ipAddress || 'N/A' },
|
||
{ "title": "Serial", "value": data.serialNumber || 'N/A' }
|
||
],
|
||
"spacing": "Small"
|
||
}
|
||
],
|
||
"actions": [
|
||
{ "type": "Action.OpenUrl", "title": "🔗 Appspace Console", "url": appspaceLink }
|
||
]
|
||
};
|
||
|
||
if (mdmFacts.length > 0) {
|
||
adaptiveCard.body.push({
|
||
"type": "Container",
|
||
"separator": true,
|
||
"items": [
|
||
{ "type": "TextBlock", "text": "**MDM Status**", "weight": "bolder", "size": "medium", "wrap": true, "color": "Accent", "spacing": "Small" },
|
||
{ "type": "FactSet", "facts": mdmFacts }
|
||
]
|
||
});
|
||
|
||
if (mdmConsoleLink) {
|
||
adaptiveCard.actions.push({
|
||
"type": "Action.OpenUrl",
|
||
"title": "🔗 Workspace ONE Console",
|
||
"url": mdmConsoleLink
|
||
});
|
||
}
|
||
}
|
||
// If no MDM data, we intentionally omit any warning block to keep the card compact and focused on the alert.
|
||
|
||
return adaptiveCard;
|
||
}
|
||
|
||
/**
|
||
* Normalize health/status field from Appspace device objects.
|
||
*/
|
||
function getDeviceHealthStatus(device) {
|
||
if (!device) return '';
|
||
return (device.status || device.healthStatus || '').toString().toUpperCase().trim();
|
||
}
|
||
|
||
/**
|
||
* Returns true for devices that are offline/lost/failed.
|
||
* Shared logic for bot queries (and potentially future webhook use).
|
||
*/
|
||
function isProblemDevice(device) {
|
||
const status = getDeviceHealthStatus(device);
|
||
return ['OFFLINE', 'LOSTCOMMUNICATION', 'FAILED'].includes(status);
|
||
}
|
||
|
||
// ======================
|
||
// HEALTHCHECK
|
||
// ======================
|
||
// Reports both HTTP server health AND Webex bot (Mercury WebSocket) health.
|
||
// Returns 503 when the bot has been detected as dead — this is what triggers
|
||
// Docker's HEALTHCHECK to mark the container unhealthy so an autoheal sidecar
|
||
// or orchestrator can restart. (For plain `docker compose` with
|
||
// `restart: unless-stopped`, the container isn't restarted on unhealthy; the
|
||
// watchdog also calls process.exit(1) after sustained failure so the compose
|
||
// restart policy will kick in.)
|
||
//
|
||
// During smoke tests (SMOKE_TEST=true), the bot is intentionally not started —
|
||
// treat the bot state as N/A rather than unhealthy so the smoke test can pass.
|
||
app.get('/health', (req, res) => {
|
||
const smokeMode = process.env.SMOKE_TEST === 'true';
|
||
const botConsideredHealthy =
|
||
smokeMode ||
|
||
!botHealth.frameworkInitialized || // startup grace: don't fail before framework is even up
|
||
botHealth.mercuryConnected !== false; // treat null (not-yet-checked) as OK
|
||
|
||
const payload = {
|
||
status: botConsideredHealthy ? 'healthy' : 'degraded',
|
||
environment: process.env.NODE_ENV || 'production',
|
||
timestamp: new Date().toISOString(),
|
||
bot: smokeMode ? 'skipped-smoke-test' : {
|
||
frameworkInitialized: botHealth.frameworkInitialized,
|
||
mercuryConnected: botHealth.mercuryConnected,
|
||
consecutiveFailures: botHealth.consecutiveFailures,
|
||
lastCheckAt: botHealth.lastCheckAt,
|
||
lastHealthyAt: botHealth.lastHealthyAt,
|
||
exiting: botHealth.exitingBecauseDead
|
||
}
|
||
};
|
||
|
||
res.status(botConsideredHealthy ? 200 : 503).json(payload);
|
||
});
|
||
|
||
// ======================
|
||
// APPSPACE WEBHOOK → Alerts (now includes UNREGISTERED)
|
||
// ======================
|
||
const { getMDMDeviceBySerial, sendMDMRebootCommand } = require('./mdm');
|
||
|
||
/**
|
||
* Query Appspace for current problem devices (Offline / LostCommunication / Failed),
|
||
* optionally narrowed by a free-text filter that matches deviceType or name.
|
||
* Shared between `offline` and `restart-offline` so they always agree on what
|
||
* counts as offline.
|
||
*
|
||
* Returns { devices, offlineDevices, apiBaseUrl } where:
|
||
* - devices: raw page returned by Appspace (used to detect 500-row truncation)
|
||
* - offlineDevices: filtered list of problem devices matching filterArg
|
||
* - apiBaseUrl: resolved base URL (for diagnostic logging in callers)
|
||
*/
|
||
async function fetchOfflineDevices(filterArg = '') {
|
||
const apiBaseUrl = process.env.APPSPACE_API_BASE_URL || process.env.APPSPACE_INSTANCE_URL || 'https://api.cloud.appspace.com';
|
||
const accessToken = await getValidAccessToken();
|
||
|
||
const response = await axios.get(`${apiBaseUrl}/api/v3/devices`, {
|
||
headers: { Authorization: `Bearer ${accessToken}` },
|
||
params: {
|
||
limit: 500,
|
||
healthStatus: 'Offline,LostCommunication,Failed',
|
||
status: 'Offline,LostCommunication,Failed'
|
||
},
|
||
timeout: 15000
|
||
});
|
||
|
||
const devices = response.data?.items || response.data || [];
|
||
let offlineDevices = devices.filter(isProblemDevice);
|
||
|
||
if (filterArg) {
|
||
const search = filterArg.toLowerCase().trim();
|
||
offlineDevices = offlineDevices.filter(d => {
|
||
const type = (d.deviceType || '').toLowerCase();
|
||
const name = (d.name || d.deviceName || '').toLowerCase();
|
||
return type.includes(search) || name.includes(search);
|
||
});
|
||
}
|
||
|
||
return { devices, offlineDevices, apiBaseUrl };
|
||
}
|
||
|
||
app.post('/webhook', async (req, res) => {
|
||
const payload = req.body;
|
||
|
||
const WEBHOOK_SECRET = process.env.WEBHOOK_SECRET;
|
||
if (WEBHOOK_SECRET) {
|
||
const receivedSecret = req.headers['x-webhook-secret'] || req.headers['x-secret'];
|
||
if (receivedSecret !== WEBHOOK_SECRET) {
|
||
logger.warn('Invalid secret header');
|
||
return res.status(401).send('Unauthorized');
|
||
}
|
||
}
|
||
|
||
const webhookType = payload.webhookType || '';
|
||
const data = payload.data || {};
|
||
|
||
// Log full payload only when debugging (can contain sensitive device/user data)
|
||
if (process.env.DEBUG_WEBHOOK === 'true' || process.env.NODE_ENV !== 'production') {
|
||
logger.debug('Appspace Webhook Received (full payload)', { payload });
|
||
} else {
|
||
logger.info('Appspace Webhook Received', { event: webhookType, device: data.deviceName || data.serialNumber || 'unknown' });
|
||
}
|
||
|
||
// ======================
|
||
// FILTER OUT PWA DEVICES
|
||
// ======================
|
||
if (data.deviceType === 'PWA' || data.deviceType?.toUpperCase() === 'PWA') {
|
||
if (isVerbose) logger.debug('Ignoring PWA device alert', { device: data.deviceName });
|
||
return res.status(200).send('Ignored PWA');
|
||
}
|
||
|
||
if (!webhookType.startsWith('DEVICE.HEALTHSTATUS.') && webhookType !== 'DEVICE.UNREGISTERED') {
|
||
if (isVerbose) logger.debug('Non-relevant event ignored');
|
||
return res.status(200).send('Ignored');
|
||
}
|
||
|
||
const eventKey = webhookType.replace('DEVICE.', '').replace('HEALTHSTATUS.', '').toUpperCase();
|
||
const isUnregistered = webhookType === 'DEVICE.UNREGISTERED';
|
||
const isProblem = isUnregistered || ['LOSTCOMMUNICATION', 'OFFLINE', 'FAILED'].includes(eventKey);
|
||
|
||
const accentColor = isProblem ? 'attention' : 'good';
|
||
const emoji = isUnregistered ? '🚫' : (isProblem ? '🔴' : '🟢');
|
||
|
||
const consoleBase = process.env.APPSPACE_CONSOLE_BASE_URL || 'https://app3.cloud.appspace.com';
|
||
const appspaceLink = `${consoleBase}/console/devices/details/overview?id=${data.deviceId}`;
|
||
|
||
// Enrich with Workspace ONE MDM (graceful; never blocks the alert)
|
||
const mdmDevice = await getMDMDeviceBySerial(data.serialNumber).catch(() => null);
|
||
const { mdmFacts, mdmConsoleLink } = buildMdmFactsAndLink(mdmDevice);
|
||
|
||
if (mdmDevice && mdmFacts.length > 0 && isVerbose) {
|
||
logger.debug('MDM data added', { device: data.deviceName });
|
||
}
|
||
|
||
const adaptiveCard = buildDeviceAlertCard({
|
||
eventKey,
|
||
data,
|
||
accentColor,
|
||
emoji,
|
||
appspaceLink,
|
||
mdmFacts,
|
||
mdmConsoleLink
|
||
});
|
||
|
||
try {
|
||
await axios.post('https://webexapis.com/v1/messages', {
|
||
roomId: process.env.WEBEX_ROOM_ID,
|
||
text: `${eventKey} on ${data.deviceName}`,
|
||
attachments: [{ contentType: "application/vnd.microsoft.card.adaptive", content: adaptiveCard }]
|
||
}, {
|
||
headers: { Authorization: `Bearer ${process.env.WEBEX_BOT_TOKEN}`, 'Content-Type': 'application/json' }
|
||
});
|
||
logger.info(`${eventKey} enriched card sent to Webex`);
|
||
} catch (err) {
|
||
logger.error('Failed to send card', { error: err.response?.data || err.message });
|
||
}
|
||
|
||
res.status(200).send('OK');
|
||
});
|
||
|
||
// ======================
|
||
// WEBEX BOT FRAMEWORK SETUP — WEBSOCKET MODE
|
||
// ======================
|
||
// Using WebSocket mode (no webhookUrl) to avoid public endpoint + rate-limit issues on restarts.
|
||
// Skipped in smoke tests (dummy token would cause noisy unhandled rejections; we only need the HTTP server + health for the smoke).
|
||
if (process.env.SMOKE_TEST !== 'true') {
|
||
framework = new Framework({
|
||
token: process.env.WEBEX_BOT_TOKEN,
|
||
// On startup, delete any stale WDM device registrations left behind by
|
||
// previously-dead-and-restarted instances. Prevents "excessive device
|
||
// registrations" errors that accumulate over the lifetime of the bot
|
||
// account when the process silently loses its Mercury socket and gets
|
||
// restarted. Safe for single-instance deployments; if you ever run
|
||
// multiple instances against the same bot token, revisit this.
|
||
removeDeviceRegistrationsOnStart: true
|
||
});
|
||
|
||
// If the framework fails to start (bad token, WebSocket handshake failure,
|
||
// Webex-side outage) the rejection would otherwise be swallowed by the global
|
||
// unhandledRejection handler and the bot would silently stay dead while
|
||
// Express keeps serving. Surface it explicitly so the failure is obvious.
|
||
framework.start().catch((err) => {
|
||
logger.error('Webex framework failed to start (bot will be unresponsive; HTTP server continues)', err);
|
||
});
|
||
|
||
framework.on('initialized', () => {
|
||
logger.info('Webex Bot Framework initialized (WebSocket mode)');
|
||
botHealth.frameworkInitialized = true;
|
||
// Seed the watchdog state so /health has something meaningful to report
|
||
// before the first watchdog tick fires.
|
||
const mercury = framework.webex?.internal?.mercury;
|
||
if (mercury) {
|
||
botHealth.mercuryConnected = !!mercury.connected;
|
||
botHealth.lastHealthyAt = new Date().toISOString();
|
||
}
|
||
});
|
||
|
||
framework.on('spawn', (bot, id, addedBy) => {
|
||
if (isVerbose) {
|
||
if (addedBy) {
|
||
logger.debug('Bot added to new space', { addedBy });
|
||
} else {
|
||
logger.debug('Bot loaded space', { title: bot.room?.title || 'Unknown' });
|
||
}
|
||
}
|
||
});
|
||
|
||
// Forward the framework's internal 'log' events into our structured logger.
|
||
// Without this, framework-internal diagnostics (device registration issues,
|
||
// membership rule denials, etc.) are lost — which is a big part of why the
|
||
// Mercury silent-death is hard to diagnose from our logs alone.
|
||
framework.on('log', (msg) => {
|
||
logger.info('[framework]', { message: msg });
|
||
});
|
||
|
||
if (isVerbose) {
|
||
logger.debug('Webex Bot Framework starting in WebSocket mode...');
|
||
}
|
||
|
||
// ======================
|
||
// MERCURY WATCHDOG
|
||
// ======================
|
||
// The Webex SDK's Mercury WebSocket (server-push event stream) can die
|
||
// silently — network blip, WDM device TTL expiry, or a Cisco-side hiccup —
|
||
// without ever emitting an error the framework surfaces. When that happens
|
||
// the HTTP paths (webhooks, message-send API) keep working but the bot
|
||
// stops receiving commands. This watchdog polls the SDK's live
|
||
// `mercury.connected` boolean; if it stays false for enough consecutive
|
||
// checks, we log an error, mark ourselves unhealthy, and eventually exit
|
||
// so Docker's restart policy brings us back with a fresh connection.
|
||
const WATCHDOG_INTERVAL_MS = 60 * 1000; // check every minute
|
||
const WATCHDOG_WARN_AFTER = 2; // 2 consecutive misses → warn (~2 min)
|
||
const WATCHDOG_EXIT_AFTER = 5; // 5 consecutive misses → exit (~5 min)
|
||
|
||
const watchdogTimer = setInterval(() => {
|
||
botHealth.lastCheckAt = new Date().toISOString();
|
||
|
||
// If the framework hasn't finished initializing yet, don't count misses
|
||
// — we're still in the startup window.
|
||
if (!botHealth.frameworkInitialized) {
|
||
return;
|
||
}
|
||
|
||
const mercury = framework.webex?.internal?.mercury;
|
||
const connected = !!(mercury && mercury.connected);
|
||
botHealth.mercuryConnected = connected;
|
||
|
||
if (connected) {
|
||
if (botHealth.consecutiveFailures > 0) {
|
||
logger.info('Mercury WebSocket recovered', {
|
||
afterFailures: botHealth.consecutiveFailures
|
||
});
|
||
}
|
||
botHealth.consecutiveFailures = 0;
|
||
botHealth.lastHealthyAt = botHealth.lastCheckAt;
|
||
return;
|
||
}
|
||
|
||
botHealth.consecutiveFailures++;
|
||
|
||
if (botHealth.consecutiveFailures === WATCHDOG_WARN_AFTER) {
|
||
logger.warn('Mercury WebSocket appears disconnected — bot may be silent', {
|
||
consecutiveFailures: botHealth.consecutiveFailures,
|
||
lastHealthyAt: botHealth.lastHealthyAt
|
||
});
|
||
} else if (botHealth.consecutiveFailures >= WATCHDOG_EXIT_AFTER && !botHealth.exitingBecauseDead) {
|
||
botHealth.exitingBecauseDead = true;
|
||
logger.error('Mercury WebSocket dead for sustained period — exiting so Docker restart policy can reconnect', {
|
||
consecutiveFailures: botHealth.consecutiveFailures,
|
||
lastHealthyAt: botHealth.lastHealthyAt
|
||
});
|
||
clearInterval(watchdogTimer);
|
||
// Give the log line + any in-flight HTTP responses a beat to flush,
|
||
// then trigger the same graceful shutdown path as SIGTERM. If that
|
||
// hangs, shutdown()'s own safety timeout forces exit.
|
||
setTimeout(() => shutdown(), 500).unref();
|
||
}
|
||
}, WATCHDOG_INTERVAL_MS);
|
||
// .unref() so this timer doesn't hold the process open by itself.
|
||
watchdogTimer.unref();
|
||
|
||
/**
|
||
* Extract the trailing filter portion of a command message, regardless of whether
|
||
* the message came from a DM (`offline ios`) or a group-space mention
|
||
* (`@appspace offline ios`).
|
||
*
|
||
* The framework sets `trigger.args = trigger.text.split(' ')`, and in group-space
|
||
* mentions `trigger.text` still has the bot's display name at the start. So we can't
|
||
* just `slice(1)` — that would take everything after the display name (including
|
||
* the command word itself) and treat it as the filter.
|
||
*
|
||
* Strategy: find the command word (case-insensitively) in the args array, then
|
||
* everything after it is the filter.
|
||
*/
|
||
function extractFilterArg(trigger, commandWord) {
|
||
const args = trigger.args || [];
|
||
const idx = args.findIndex(a => (a || '').toLowerCase() === commandWord.toLowerCase());
|
||
if (idx === -1 || idx === args.length - 1) return '';
|
||
return args.slice(idx + 1).join(' ').toLowerCase().trim();
|
||
}
|
||
|
||
// NOTE: keep as a string phrase (NOT a regex). The framework compiles string
|
||
// phrases into `/(^| )<phrase>($| )/i`, which:
|
||
// - correctly matches after the bot's display name in group-space mentions
|
||
// (e.g. `@appspace offline ios` → trigger.text = "Appspace offline ios")
|
||
// - uses SPACE delimiters (not `\b`), so it will NOT match the substring
|
||
// "offline" inside "restart-offline" (preceded by `-`, not space).
|
||
// A regex like /^offline\b/i is tested directly against trigger.text and would
|
||
// fail on any mentioned message because the display name comes first.
|
||
framework.hears('offline', async (bot, trigger) => {
|
||
if (isVerbose) {
|
||
logger.debug('offline command received', { user: trigger.person?.displayName || 'Unknown' });
|
||
}
|
||
|
||
const filterArg = extractFilterArg(trigger, 'offline');
|
||
|
||
await bot.say({ markdown: '🔍 Querying current offline / lost devices from Appspace...' });
|
||
|
||
// Pre-resolve for diagnostic logging in the catch block.
|
||
let apiBaseUrl = process.env.APPSPACE_API_BASE_URL || process.env.APPSPACE_INSTANCE_URL || 'https://api.cloud.appspace.com';
|
||
|
||
try {
|
||
const fetched = await fetchOfflineDevices(filterArg);
|
||
const { devices, offlineDevices } = fetched;
|
||
apiBaseUrl = fetched.apiBaseUrl;
|
||
|
||
if (offlineDevices.length === 0) {
|
||
const msg = filterArg
|
||
? `✅ No **${filterArg}** devices are currently offline.`
|
||
: '✅ All devices are currently online or in sync.';
|
||
return bot.say({ markdown: msg });
|
||
}
|
||
|
||
const consoleBase = process.env.APPSPACE_CONSOLE_BASE_URL || 'https://app3.cloud.appspace.com';
|
||
const consoleUrl = `${consoleBase}/console/devices`;
|
||
|
||
// Enrich the devices we will display (cap at 30) with MDM + per-device Appspace links.
|
||
// Using a rich Markdown list (instead of table) so everything is consolidated per device
|
||
// and links render properly as clickable items.
|
||
const MAX_ROWS = 30;
|
||
const displayDevices = offlineDevices.slice(0, MAX_ROWS);
|
||
const enriched = await Promise.all(displayDevices.map(async (d) => {
|
||
const mdmDevice = await getMDMDeviceBySerial(d.serialNumber).catch(() => null);
|
||
let mdmInfo = '';
|
||
let mdmConsoleLink = null;
|
||
if (mdmDevice) {
|
||
const { mdmFacts, mdmConsoleLink: link } = buildMdmFactsAndLink(mdmDevice);
|
||
mdmConsoleLink = link;
|
||
if (mdmFacts.length > 0) {
|
||
mdmInfo = mdmFacts.map(f => `${f.title}: ${f.value}`).join(' | ');
|
||
}
|
||
}
|
||
const devId = d.deviceId || d.id || '';
|
||
const appspaceLink = devId ? `${consoleBase}/console/devices/details/overview?id=${devId}` : null;
|
||
|
||
return {
|
||
...d,
|
||
mdmInfo,
|
||
mdmConsoleLink,
|
||
appspaceLink
|
||
};
|
||
}));
|
||
|
||
// Group by location for better readability. Each device gets:
|
||
// - Name + basic Appspace info on first line (under bullet)
|
||
// - MDM info on second indented line
|
||
// - Links on third indented line
|
||
// No deep nesting; only bullet the device line, indent the rest.
|
||
const grouped = {};
|
||
enriched.forEach(e => {
|
||
const loc = e.locationName || 'Unknown Location';
|
||
if (!grouped[loc]) grouped[loc] = [];
|
||
grouped[loc].push(e);
|
||
});
|
||
|
||
// Webex caps a single message at 7439 characters before encryption.
|
||
// Build the body incrementally and stop appending devices once we approach the
|
||
// budget, leaving headroom for the header + console-link footer + a truncation note.
|
||
// The user can always click through to the full console list.
|
||
const WEBEX_BODY_BUDGET = 6500;
|
||
const filterNote = filterArg ? ` (filtered: ${filterArg})` : '';
|
||
const limitNote = devices.length >= 500 ? ' (results may be truncated — see console for full list)' : '';
|
||
|
||
let devicesList = '';
|
||
let renderedCount = 0;
|
||
let truncated = false;
|
||
|
||
outer: for (const loc of Object.keys(grouped).sort()) {
|
||
const locHeader = `**${loc}**\n`;
|
||
// If even the location header won't fit, stop entirely.
|
||
if (devicesList.length + locHeader.length > WEBEX_BODY_BUDGET) {
|
||
truncated = true;
|
||
break;
|
||
}
|
||
devicesList += locHeader;
|
||
|
||
for (const e of grouped[loc]) {
|
||
const name = (e.name || e.deviceName || 'Unknown');
|
||
const status = getDeviceHealthStatus(e) || 'Unknown';
|
||
const ip = e.ipAddress || 'N/A';
|
||
const type = e.deviceType || '—';
|
||
|
||
let entry = `- **${name}** (${status}) - IP: ${ip} Type: ${type}\n`;
|
||
entry += e.mdmInfo ? ` MDM: ${e.mdmInfo}\n` : ` No MDM data\n`;
|
||
let linksLine = '';
|
||
if (e.appspaceLink) linksLine += `[🔗 Appspace Console](${e.appspaceLink})`;
|
||
if (e.mdmConsoleLink) {
|
||
if (linksLine) linksLine += ' ';
|
||
linksLine += `[🔗 Workspace ONE Console](${e.mdmConsoleLink})`;
|
||
}
|
||
if (linksLine) entry += ` ${linksLine}\n`;
|
||
entry += '\n';
|
||
|
||
if (devicesList.length + entry.length > WEBEX_BODY_BUDGET) {
|
||
truncated = true;
|
||
break outer;
|
||
}
|
||
|
||
devicesList += entry;
|
||
renderedCount++;
|
||
}
|
||
}
|
||
|
||
// Truncation note covers both (a) enrichment cap (MAX_ROWS < total offline)
|
||
// and (b) character-budget truncation that stopped us mid-render.
|
||
const notShown = offlineDevices.length - renderedCount;
|
||
if (notShown > 0 || truncated) {
|
||
devicesList += `... (${notShown} more not shown, see full list in Appspace Console)\n`;
|
||
}
|
||
|
||
const message = `**📊 Offline Devices Snapshot (${offlineDevices.length})${filterNote}${limitNote}**\n\n` +
|
||
devicesList +
|
||
`[🔗 Open Devices in Appspace Console](${consoleUrl})`;
|
||
|
||
if (isVerbose) {
|
||
logger.debug(`Offline query returned ${offlineDevices.length} matching devices`);
|
||
}
|
||
|
||
await bot.say({ markdown: message });
|
||
|
||
} catch (err) {
|
||
const attemptedUrl = `${apiBaseUrl}/api/v3/devices`;
|
||
logger.error('Offline query failed', {
|
||
attemptedUrl,
|
||
error: err.message,
|
||
code: err.code
|
||
});
|
||
bot.say({ markdown: `⚠️ Failed to query offline devices.\n\n${err.message || 'Unknown error'}` });
|
||
}
|
||
});
|
||
|
||
// restart-offline [filter]
|
||
// For every currently-offline (Offline / LostCommunication / Failed) Appspace device
|
||
// whose serial maps to a Workspace ONE record, send a SoftReset (reboot) command.
|
||
// Hard-capped at MAX_RESTART_BATCH per invocation; use a filter to narrow further.
|
||
// Always re-queries Appspace at execution time, so devices that came back online
|
||
// since the user last looked will NOT be rebooted.
|
||
// See note on the 'offline' handler above: string phrases handle mentions
|
||
// correctly (framework wraps them as `(^| )restart-offline($| )/i`).
|
||
framework.hears('restart-offline', async (bot, trigger) => {
|
||
const MAX_RESTART_BATCH = 50;
|
||
const CONCURRENCY = 3;
|
||
|
||
const filterArg = extractFilterArg(trigger, 'restart-offline');
|
||
|
||
const userLabel = trigger.person?.emails?.[0] || trigger.person?.displayName || 'Unknown';
|
||
logger.info('restart-offline invoked', { user: userLabel, filter: filterArg || '(none)' });
|
||
|
||
await bot.say({ markdown: `🔁 Querying current offline devices${filterArg ? ` matching \`${filterArg}\`` : ''} from Appspace...` });
|
||
|
||
let offlineDevices;
|
||
try {
|
||
const fetched = await fetchOfflineDevices(filterArg);
|
||
offlineDevices = fetched.offlineDevices;
|
||
} catch (err) {
|
||
logger.error('restart-offline: Appspace query failed', err);
|
||
return bot.say({ markdown: `⚠️ Failed to query offline devices from Appspace.\n\n${err.message || 'Unknown error'}` });
|
||
}
|
||
|
||
if (offlineDevices.length === 0) {
|
||
return bot.say({ markdown: filterArg
|
||
? `✅ No **${filterArg}** devices are currently offline. Nothing to restart.`
|
||
: '✅ No devices are currently offline. Nothing to restart.' });
|
||
}
|
||
|
||
if (offlineDevices.length > MAX_RESTART_BATCH) {
|
||
return bot.say({ markdown:
|
||
`🛑 **${offlineDevices.length}** offline devices match — that exceeds the safety cap of **${MAX_RESTART_BATCH}** per invocation.\n\n` +
|
||
`Please narrow with a filter (e.g. \`restart-offline ios\`, \`restart-offline windows\`, or part of a device name) and try again.`
|
||
});
|
||
}
|
||
|
||
// Resolve each offline device to its WS1 numeric Id (needed for the SoftReset command).
|
||
const candidates = await Promise.all(offlineDevices.map(async (d) => {
|
||
const mdmDevice = await getMDMDeviceBySerial(d.serialNumber).catch(() => null);
|
||
return {
|
||
name: d.name || d.deviceName || 'Unknown',
|
||
location: d.locationName || 'Unknown',
|
||
serial: d.serialNumber || '(no serial)',
|
||
mdmId: mdmDevice?.Id?.Value || null
|
||
};
|
||
}));
|
||
|
||
const withMdm = candidates.filter(c => c.mdmId);
|
||
const withoutMdm = candidates.filter(c => !c.mdmId);
|
||
|
||
if (withMdm.length === 0) {
|
||
return bot.say({ markdown:
|
||
`⚠️ Found **${offlineDevices.length}** offline device(s), but none have a Workspace ONE record (matched by serial). Nothing to restart.`
|
||
});
|
||
}
|
||
|
||
await bot.say({ markdown:
|
||
`🔁 Sending **SoftReset** to **${withMdm.length}** device(s) via Workspace ONE...` +
|
||
(withoutMdm.length > 0 ? `\n_(${withoutMdm.length} offline device(s) have no WS1 record — skipping those.)_` : '')
|
||
});
|
||
|
||
// Bounded concurrency so we don't fire 50 reboots at WS1 in parallel.
|
||
const results = [];
|
||
for (let i = 0; i < withMdm.length; i += CONCURRENCY) {
|
||
const batch = withMdm.slice(i, i + CONCURRENCY);
|
||
const batchResults = await Promise.all(batch.map(async (c) => {
|
||
const r = await sendMDMRebootCommand(c.mdmId);
|
||
return { ...c, ...r };
|
||
}));
|
||
results.push(...batchResults);
|
||
}
|
||
|
||
const successes = results.filter(r => r.success);
|
||
const failures = results.filter(r => !r.success);
|
||
|
||
logger.info('restart-offline executed', {
|
||
user: userLabel,
|
||
filter: filterArg || '(none)',
|
||
matched: offlineDevices.length,
|
||
withMdm: withMdm.length,
|
||
withoutMdm: withoutMdm.length,
|
||
succeeded: successes.length,
|
||
failed: failures.length
|
||
});
|
||
|
||
// Webex 7439-char limit applies here too. Keep the report compact.
|
||
const WEBEX_BODY_BUDGET = 6500;
|
||
let msg = `**🔁 Restart Complete**${filterArg ? ` _(filter: \`${filterArg}\`)_` : ''}\n\n` +
|
||
`- Offline matched: **${offlineDevices.length}**\n` +
|
||
`- Restart command sent: **${successes.length}**\n` +
|
||
(failures.length > 0 ? `- Failed: **${failures.length}**\n` : '') +
|
||
(withoutMdm.length > 0 ? `- Skipped (no WS1 record): **${withoutMdm.length}**\n` : '') +
|
||
`\n_Note: WS1 only queues the command; an iOS device must be Supervised for SoftReset to actually execute._\n`;
|
||
|
||
if (failures.length > 0) {
|
||
msg += `\n**Failures:**\n`;
|
||
let failureLines = '';
|
||
let shown = 0;
|
||
for (const f of failures) {
|
||
const line = `- ${f.name} (${f.serial}): ${f.error || `HTTP ${f.status || 'unknown'}`}\n`;
|
||
if (msg.length + failureLines.length + line.length > WEBEX_BODY_BUDGET) break;
|
||
failureLines += line;
|
||
shown++;
|
||
}
|
||
msg += failureLines;
|
||
if (shown < failures.length) {
|
||
msg += `- ... and ${failures.length - shown} more failure(s) (see service logs)\n`;
|
||
}
|
||
}
|
||
|
||
if (withoutMdm.length > 0 && msg.length < WEBEX_BODY_BUDGET - 500) {
|
||
msg += `\n**Skipped (no WS1 record):**\n`;
|
||
let skipLines = '';
|
||
let shown = 0;
|
||
for (const s of withoutMdm) {
|
||
const line = `- ${s.name} at ${s.location} (serial: ${s.serial})\n`;
|
||
if (msg.length + skipLines.length + line.length > WEBEX_BODY_BUDGET) break;
|
||
skipLines += line;
|
||
shown++;
|
||
}
|
||
msg += skipLines;
|
||
if (shown < withoutMdm.length) {
|
||
msg += `- ... and ${withoutMdm.length - shown} more (see service logs)\n`;
|
||
}
|
||
}
|
||
|
||
await bot.say({ markdown: msg });
|
||
});
|
||
|
||
framework.hears('help', (bot) => {
|
||
bot.say({ markdown:
|
||
'**Commands:**\n' +
|
||
'• `offline [filter]` — Current offline / Lost / Failed devices from Appspace (optional name/type filter, e.g. `offline ios`)\n' +
|
||
'• `restart-offline [filter]` — Send a SoftReset (reboot) via Workspace ONE to every currently offline device that has a WS1 record. Optional filter narrows the set. Capped at 50 per invocation.\n' +
|
||
'• `help` — This message\n\n' +
|
||
'See README for setup, Docker usage, and debug flags.'
|
||
});
|
||
});
|
||
}
|
||
|
||
server = app.listen(PORT, () => {
|
||
logger.info(`🚀 Server listening on port ${PORT}`);
|
||
logger.info(` Appspace webhook: POST /webhook`);
|
||
logger.info(` Webex bot: WebSocket mode (no public HTTP webhook registered)`);
|
||
if (process.env.PUBLIC_HOST) {
|
||
logger.info(` Public host: ${process.env.PUBLIC_HOST}`);
|
||
}
|
||
logger.info(' Press Ctrl+C or send SIGTERM for graceful shutdown (Docker-friendly)');
|
||
});
|
||
|
||
// Register signal handlers for graceful shutdown (critical for Docker and orchestrators)
|
||
process.on('SIGTERM', () => {
|
||
logger.info('📡 Received SIGTERM (Docker stop / orchestrator)');
|
||
shutdown();
|
||
});
|
||
|
||
process.on('SIGINT', () => {
|
||
logger.info('📡 Received SIGINT (Ctrl+C)');
|
||
shutdown();
|
||
});
|