jmcqueen
49bb5fd719
Add in-process framework restart, Mercury event listeners, and safer command reply handling to recover from silent WebSocket death without waiting for a manual container restart. Co-authored-by: Cursor <cursoragent@cursor.com>
1104 lines
45 KiB
JavaScript
1104 lines
45 KiB
JavaScript
require('dotenv').config();
|
||
const express = require('express');
|
||
const axios = require('axios');
|
||
const Framework = require('webex-node-bot-framework');
|
||
|
||
const app = express();
|
||
const PORT = process.env.PORT || 3000;
|
||
|
||
// ======================
|
||
// STARTUP VALIDATION (fail fast on missing critical config)
|
||
// ======================
|
||
const REQUIRED_ENV = [
|
||
'WEBEX_BOT_TOKEN',
|
||
'WEBEX_ROOM_ID',
|
||
'APPSPACE_INSTANCE_URL',
|
||
'APPSPACE_SUBJECT_ID',
|
||
'APPSPACE_REFRESH_TOKEN',
|
||
'APPSPACE_API_BASE_URL'
|
||
];
|
||
|
||
const missing = REQUIRED_ENV.filter(k => !process.env[k]);
|
||
if (missing.length > 0) {
|
||
console.error('❌ Missing required environment variables:');
|
||
missing.forEach(k => console.error(` - ${k}`));
|
||
console.error(' See .env.example for details.');
|
||
process.exit(1);
|
||
}
|
||
|
||
// Logging configuration for Docker / production friendliness
|
||
const isVerbose = process.env.DEBUG === 'true' || process.env.NODE_ENV !== 'production';
|
||
const useJsonLogs = process.env.LOG_FORMAT === 'json' || process.env.NODE_ENV === 'production';
|
||
|
||
/**
|
||
* Minimal structured logger.
|
||
* - Human readable (with emojis) in dev
|
||
* - JSON lines when LOG_FORMAT=json or NODE_ENV=production (great for Docker log collectors)
|
||
*/
|
||
const logger = {
|
||
info: (msg, meta = {}) => log('info', msg, meta),
|
||
warn: (msg, meta = {}) => log('warn', msg, meta),
|
||
error: (msg, meta = {}) => log('error', msg, meta),
|
||
debug: (msg, meta = {}) => { if (isVerbose) log('debug', msg, meta); }
|
||
};
|
||
|
||
// Normalize whatever was passed as `meta` into a plain object suitable for JSON
|
||
// logging. Errors have no enumerable own properties, so JSON.stringify(err) ⇒ "{}".
|
||
// We pull out the useful bits (name/message/stack + axios response shape) so failures
|
||
// are actually visible in the log stream.
|
||
function normalizeMeta(meta) {
|
||
if (meta == null) return {};
|
||
if (meta instanceof Error) {
|
||
const out = { error: meta.message, errorName: meta.name, stack: meta.stack };
|
||
if (meta.code) out.code = meta.code;
|
||
if (meta.response) {
|
||
out.responseStatus = meta.response.status;
|
||
out.responseData = meta.response.data;
|
||
}
|
||
return out;
|
||
}
|
||
if (typeof meta !== 'object') return { value: meta };
|
||
return meta;
|
||
}
|
||
|
||
function log(level, msg, meta = {}) {
|
||
const timestamp = new Date().toISOString();
|
||
const normMeta = normalizeMeta(meta);
|
||
if (useJsonLogs) {
|
||
const entry = { timestamp, level, msg, ...normMeta };
|
||
// Remove undefined
|
||
Object.keys(entry).forEach(k => entry[k] === undefined && delete entry[k]);
|
||
console.log(JSON.stringify(entry));
|
||
} else {
|
||
const emoji = level === 'error' ? '❌' : level === 'warn' ? '⚠️' : level === 'debug' ? '🐛' : 'ℹ️';
|
||
const metaStr = Object.keys(normMeta).length ? ' ' + JSON.stringify(normMeta) : '';
|
||
const out = level === 'error' ? console.error : console.log;
|
||
out(`${emoji} ${msg}${metaStr}`);
|
||
}
|
||
}
|
||
|
||
// Will be assigned when the server starts listening
|
||
let server;
|
||
let framework;
|
||
|
||
// ======================
|
||
// BOT LIVENESS STATE (populated by the Mercury watchdog below)
|
||
// ======================
|
||
// Represents whether the Webex bot's WebSocket (Mercury) transport is currently
|
||
// healthy. Read by /health so container orchestrators can observe the state.
|
||
// Set to `true` initially so we don't fail the healthcheck during the startup
|
||
// window before the framework has finished initializing.
|
||
const botHealth = {
|
||
mercuryConnected: null, // last-observed value of webex.internal.mercury.connected (null = not-yet-checked)
|
||
consecutiveFailures: 0, // consecutive watchdog checks that saw a dead socket
|
||
lastCheckAt: null, // ISO timestamp of last watchdog check
|
||
lastHealthyAt: null, // ISO timestamp of last healthy check (for staleness reporting)
|
||
frameworkInitialized: false, // set true on framework 'initialized' event
|
||
exitingBecauseDead: false, // set true when the watchdog is about to exit the process
|
||
lastReconnectAttemptAt: null, // ISO timestamp of last in-process framework restart attempt
|
||
reconnectAttempts: 0 // total reconnect attempts this process lifetime
|
||
};
|
||
|
||
// Prevents overlapping framework.stop()/start() calls from the watchdog and Mercury events.
|
||
let reconnectInProgress = false;
|
||
|
||
// Graceful shutdown function (used by signals and crash handlers)
|
||
function shutdown(force = false) {
|
||
logger.info('🛑 Graceful shutdown initiated...');
|
||
|
||
const exit = (code = 0) => {
|
||
logger.info(`👋 Process exiting with code ${code}`);
|
||
process.exit(code);
|
||
};
|
||
|
||
// Stop the Webex framework (important for WebSocket mode to clean up device registration)
|
||
if (typeof framework !== 'undefined' && framework.stop) {
|
||
framework.stop().then(() => {
|
||
logger.info('✅ Webex framework stopped');
|
||
if (server) {
|
||
server.close(() => {
|
||
logger.info('✅ HTTP server closed');
|
||
exit(0);
|
||
});
|
||
} else {
|
||
exit(0);
|
||
}
|
||
}).catch((err) => {
|
||
logger.error('Error stopping framework:', err);
|
||
if (server) server.close(() => exit(1));
|
||
else exit(1);
|
||
});
|
||
} else if (server) {
|
||
server.close(() => {
|
||
logger.info('✅ HTTP server closed');
|
||
exit(0);
|
||
});
|
||
} else {
|
||
exit(0);
|
||
}
|
||
|
||
// ALWAYS arm a safety timeout. If any step above hangs (e.g. framework.stop()
|
||
// never resolves), we still need to exit before Docker's 10s SIGKILL. When called
|
||
// from a crash handler (`force=true`) the process is already unhealthy, so exit
|
||
// faster to minimize the window where we're wedged.
|
||
setTimeout(() => {
|
||
logger.error('⏱️ Graceful shutdown timed out. Forcing exit.');
|
||
exit(1);
|
||
}, force ? 3000 : 8000).unref();
|
||
}
|
||
|
||
// Basic crash handlers for container environments (Docker/K8s will restart on non-zero exit)
|
||
// Uncaught is fatal -> shutdown.
|
||
// UnhandledRejection (e.g. from Webex framework with bad creds or transient issues) just log; don't kill the main server.
|
||
process.on('uncaughtException', (err) => {
|
||
logger.error('Uncaught Exception. Initiating shutdown...', err);
|
||
shutdown(true);
|
||
});
|
||
|
||
process.on('unhandledRejection', (reason) => {
|
||
// `reason` is most often an Error but can be anything; the logger normalizes it.
|
||
// If something rejected with a non-Error value, wrap it so we still get a stack-y view.
|
||
const payload = reason instanceof Error
|
||
? reason
|
||
: { reason: typeof reason === 'object' ? JSON.stringify(reason) : String(reason) };
|
||
logger.error('Unhandled Rejection (logged, continuing)...', payload);
|
||
// Do not call shutdown - keep the HTTP server running (e.g. Webex bot errors shouldn't kill webhook path)
|
||
});
|
||
|
||
app.use(express.json());
|
||
|
||
// ======================
|
||
// APPSPACE TOKEN MANAGEMENT (with cooldown + longer timeout)
|
||
// ======================
|
||
let currentAccessToken = null;
|
||
let tokenExpiresAt = 0;
|
||
let lastRefreshAttempt = 0;
|
||
let inFlightAppspaceTokenPromise = null;
|
||
|
||
function invalidateAppspaceToken() {
|
||
currentAccessToken = null;
|
||
tokenExpiresAt = 0;
|
||
}
|
||
|
||
async function getValidAccessToken() {
|
||
const now = Math.floor(Date.now() / 1000);
|
||
|
||
// Use cached token if still valid (2-minute safety buffer)
|
||
if (currentAccessToken && now < tokenExpiresAt - 120) {
|
||
return currentAccessToken;
|
||
}
|
||
|
||
// Coalesce concurrent callers (e.g. burst webhooks / bot commands at startup)
|
||
// so we never fire more than one refresh in parallel.
|
||
if (inFlightAppspaceTokenPromise) return inFlightAppspaceTokenPromise;
|
||
|
||
// Prevent hammering the endpoint (minimum 10 seconds between refresh attempts)
|
||
if (now - lastRefreshAttempt < 10) {
|
||
logger.debug('Token refresh attempted too recently — using last known token if available');
|
||
if (currentAccessToken) return currentAccessToken;
|
||
}
|
||
|
||
lastRefreshAttempt = now;
|
||
|
||
const instanceUrl = process.env.APPSPACE_INSTANCE_URL;
|
||
if (!instanceUrl) {
|
||
throw new Error('APPSPACE_INSTANCE_URL is not set in .env');
|
||
}
|
||
|
||
inFlightAppspaceTokenPromise = (async () => {
|
||
logger.info('Refreshing Appspace access token...');
|
||
try {
|
||
const response = await axios.post(`${instanceUrl}/api/v3/authorization/token`, {
|
||
subjectType: "Application",
|
||
subjectId: process.env.APPSPACE_SUBJECT_ID,
|
||
grantType: "refreshToken",
|
||
refreshToken: process.env.APPSPACE_REFRESH_TOKEN
|
||
}, {
|
||
headers: { 'Content-Type': 'application/json' },
|
||
timeout: 30000 // Increased to 30 seconds
|
||
});
|
||
|
||
const data = response.data;
|
||
currentAccessToken = data.accessToken;
|
||
tokenExpiresAt = Math.floor(Date.now() / 1000) + (data.expiresIn || 3600);
|
||
|
||
logger.info('Appspace access token refreshed successfully', { expiresIn: data.expiresIn || 3600 });
|
||
|
||
return currentAccessToken;
|
||
} catch (err) {
|
||
logger.error('Token refresh failed', { error: err.response?.data || err.message });
|
||
const status = err.response?.status;
|
||
if (status === 401 || status === 403) {
|
||
invalidateAppspaceToken();
|
||
}
|
||
if (err.code === 'ECONNABORTED') {
|
||
logger.warn('Request timed out. Appspace token endpoint may be rate-limited.');
|
||
}
|
||
throw new Error('Could not obtain valid Appspace access token. Try again in 30-60 seconds.');
|
||
}
|
||
})().finally(() => {
|
||
inFlightAppspaceTokenPromise = null;
|
||
});
|
||
|
||
return inFlightAppspaceTokenPromise;
|
||
}
|
||
|
||
// ======================
|
||
// UTILITIES
|
||
// ======================
|
||
/**
|
||
* Normalize and format MDM timestamps (LastSystemSampleTime, LastSeen, etc.)
|
||
* Handles missing 'Z' suffix from some WS1/Appspace responses and formats in
|
||
* Eastern Time (America/New_York, DST-aware — so it renders as EST or EDT
|
||
* automatically depending on the date).
|
||
*/
|
||
function formatMdmTimestamp(ts) {
|
||
if (!ts) return 'Unknown';
|
||
let timestamp = ts.toString().trim();
|
||
// Some backends omit Z on what is effectively UTC; append if it looks like it needs it
|
||
if (!timestamp.endsWith('Z') && !timestamp.includes('+') && timestamp.includes('-')) {
|
||
timestamp += 'Z';
|
||
}
|
||
const date = new Date(timestamp);
|
||
if (isNaN(date.getTime())) {
|
||
return timestamp;
|
||
}
|
||
return date.toLocaleString('en-US', {
|
||
timeZone: 'America/New_York',
|
||
month: 'numeric',
|
||
day: 'numeric',
|
||
year: 'numeric',
|
||
hour: 'numeric',
|
||
minute: '2-digit',
|
||
hour12: true
|
||
});
|
||
}
|
||
|
||
/**
|
||
* Extract a consistent set of MDM facts + Workspace ONE console link from a device record.
|
||
* Handles the quirky casing and nested Id.Value shape in WS1 responses.
|
||
* IMPORTANT: Use WS1_CONSOLE_BASE_URL (not WS1_BASE_URL) for the link base, as the API hostname (as....awmdm.com)
|
||
* is different from the console hostname (cn....awmdm.com) and will produce broken links.
|
||
*/
|
||
function buildMdmFactsAndLink(mdmDevice) {
|
||
if (!mdmDevice) {
|
||
return { mdmFacts: [], mdmConsoleLink: null };
|
||
}
|
||
|
||
const model = mdmDevice.Model || mdmDevice.model || 'Unknown';
|
||
const os = mdmDevice.OperatingSystem || 'Unknown';
|
||
const compliance = mdmDevice.ComplianceStatus || mdmDevice.complianceStatus || 'Unknown';
|
||
|
||
const lastSampleTime = formatMdmTimestamp(mdmDevice.LastSystemSampleTime);
|
||
const lastSeenDisplay = formatMdmTimestamp(mdmDevice.LastSeen);
|
||
|
||
const deviceId = mdmDevice.Id?.Value || mdmDevice.Uuid || mdmDevice.id || '';
|
||
let mdmConsoleLink = null;
|
||
if (deviceId) {
|
||
// Always use the console hostname (e.g. cn1896.awmdm.com), not the API hostname (e.g. as1896.awmdm.com)
|
||
// WS1_BASE_URL is the API server; WS1_CONSOLE_BASE_URL (or equivalent) is for the web UI links.
|
||
const ws1Base = process.env.WS1_CONSOLE_BASE_URL || 'https://cn1896.awmdm.com';
|
||
mdmConsoleLink = `${ws1Base}/AirWatch/#/AirWatch/Device/Details/Summary/${deviceId}`;
|
||
}
|
||
|
||
const mdmFacts = [];
|
||
if (model && model !== 'Unknown') mdmFacts.push({ "title": "Model", "value": model });
|
||
if (os && os !== 'Unknown') mdmFacts.push({ "title": "OS Version", "value": os });
|
||
if (compliance && compliance !== 'Unknown') mdmFacts.push({ "title": "Compliance", "value": compliance });
|
||
if (lastSampleTime && lastSampleTime !== 'Unknown') mdmFacts.push({ "title": "Last Sample (ET)", "value": lastSampleTime });
|
||
if (lastSeenDisplay && lastSeenDisplay !== 'Unknown') mdmFacts.push({ "title": "Last Seen (ET)", "value": lastSeenDisplay });
|
||
|
||
return { mdmFacts, mdmConsoleLink };
|
||
}
|
||
|
||
/**
|
||
* Construct the complete Adaptive Card payload for Webex.
|
||
* Keeps presentation logic out of the request handler.
|
||
*
|
||
* Note: Webex currently supports a maximum of Adaptive Cards 1.3
|
||
* (1.4+ is in the engineering backlog with no ETA as of 2025/2026).
|
||
* See: https://developer.webex.com/docs/buttons-and-cards
|
||
* Schema explorer: https://adaptivecards.io/explorer/
|
||
*/
|
||
function buildDeviceAlertCard({ eventKey, data, accentColor, emoji, appspaceLink, mdmFacts = [], mdmConsoleLink = null }) {
|
||
const adaptiveCard = {
|
||
"$schema": "http://adaptivecards.io/schemas/adaptive-card.json",
|
||
"type": "AdaptiveCard",
|
||
"version": "1.3", // Webex currently supports a maximum of Adaptive Cards 1.3 (1.4+ is in backlog with no ETA)
|
||
"body": [
|
||
{
|
||
"type": "Container",
|
||
"style": accentColor,
|
||
"bleed": true,
|
||
"items": [
|
||
{ "type": "TextBlock", "text": `${emoji} Appspace Device Alert`, "weight": "bolder", "size": "medium", "wrap": true }
|
||
]
|
||
},
|
||
{
|
||
"type": "FactSet",
|
||
"facts": [
|
||
{ "title": "Event", "value": eventKey },
|
||
{ "title": "Device", "value": data.deviceName || 'Unknown' },
|
||
{ "title": "Location", "value": data.locationName || 'Unknown' },
|
||
{ "title": "Type", "value": data.deviceType || 'N/A' },
|
||
{ "title": "IP", "value": data.ipAddress || 'N/A' },
|
||
{ "title": "Serial", "value": data.serialNumber || 'N/A' }
|
||
],
|
||
"spacing": "Small"
|
||
}
|
||
],
|
||
"actions": [
|
||
{ "type": "Action.OpenUrl", "title": "🔗 Appspace Console", "url": appspaceLink }
|
||
]
|
||
};
|
||
|
||
if (mdmFacts.length > 0) {
|
||
adaptiveCard.body.push({
|
||
"type": "Container",
|
||
"separator": true,
|
||
"items": [
|
||
{ "type": "TextBlock", "text": "**MDM Status**", "weight": "bolder", "size": "medium", "wrap": true, "color": "Accent", "spacing": "Small" },
|
||
{ "type": "FactSet", "facts": mdmFacts }
|
||
]
|
||
});
|
||
|
||
if (mdmConsoleLink) {
|
||
adaptiveCard.actions.push({
|
||
"type": "Action.OpenUrl",
|
||
"title": "🔗 Workspace ONE Console",
|
||
"url": mdmConsoleLink
|
||
});
|
||
}
|
||
}
|
||
// If no MDM data, we intentionally omit any warning block to keep the card compact and focused on the alert.
|
||
|
||
return adaptiveCard;
|
||
}
|
||
|
||
/**
|
||
* Normalize health/status field from Appspace device objects.
|
||
*/
|
||
function getDeviceHealthStatus(device) {
|
||
if (!device) return '';
|
||
return (device.status || device.healthStatus || '').toString().toUpperCase().trim();
|
||
}
|
||
|
||
/**
|
||
* Returns true for devices that are offline/lost/failed.
|
||
* Shared logic for bot queries (and potentially future webhook use).
|
||
*/
|
||
function isProblemDevice(device) {
|
||
const status = getDeviceHealthStatus(device);
|
||
return ['OFFLINE', 'LOSTCOMMUNICATION', 'FAILED'].includes(status);
|
||
}
|
||
|
||
// ======================
|
||
// HEALTHCHECK
|
||
// ======================
|
||
// Reports both HTTP server health AND Webex bot (Mercury WebSocket) health.
|
||
// Returns 503 when the bot has been detected as dead — this is what triggers
|
||
// Docker's HEALTHCHECK to mark the container unhealthy so an autoheal sidecar
|
||
// or orchestrator can restart. (For plain `docker compose` with
|
||
// `restart: unless-stopped`, the container isn't restarted on unhealthy; the
|
||
// watchdog also calls process.exit(1) after sustained failure so the compose
|
||
// restart policy will kick in.)
|
||
//
|
||
// During smoke tests (SMOKE_TEST=true), the bot is intentionally not started —
|
||
// treat the bot state as N/A rather than unhealthy so the smoke test can pass.
|
||
app.get('/health', (req, res) => {
|
||
const smokeMode = process.env.SMOKE_TEST === 'true';
|
||
const botConsideredHealthy =
|
||
smokeMode ||
|
||
!botHealth.frameworkInitialized || // startup grace: don't fail before framework is even up
|
||
botHealth.mercuryConnected !== false; // treat null (not-yet-checked) as OK
|
||
|
||
const payload = {
|
||
status: botConsideredHealthy ? 'healthy' : 'degraded',
|
||
environment: process.env.NODE_ENV || 'production',
|
||
timestamp: new Date().toISOString(),
|
||
bot: smokeMode ? 'skipped-smoke-test' : {
|
||
frameworkInitialized: botHealth.frameworkInitialized,
|
||
mercuryConnected: botHealth.mercuryConnected,
|
||
consecutiveFailures: botHealth.consecutiveFailures,
|
||
lastCheckAt: botHealth.lastCheckAt,
|
||
lastHealthyAt: botHealth.lastHealthyAt,
|
||
lastReconnectAttemptAt: botHealth.lastReconnectAttemptAt,
|
||
reconnectAttempts: botHealth.reconnectAttempts,
|
||
exiting: botHealth.exitingBecauseDead
|
||
}
|
||
};
|
||
|
||
res.status(botConsideredHealthy ? 200 : 503).json(payload);
|
||
});
|
||
|
||
// ======================
|
||
// APPSPACE WEBHOOK → Alerts (now includes UNREGISTERED)
|
||
// ======================
|
||
const { getMDMDeviceBySerial, sendMDMRebootCommand } = require('./mdm');
|
||
|
||
/**
|
||
* Query Appspace for current problem devices (Offline / LostCommunication / Failed),
|
||
* optionally narrowed by a free-text filter that matches deviceType or name.
|
||
* Shared between `offline` and `restart-offline` so they always agree on what
|
||
* counts as offline.
|
||
*
|
||
* Returns { devices, offlineDevices, apiBaseUrl } where:
|
||
* - devices: raw page returned by Appspace (used to detect 500-row truncation)
|
||
* - offlineDevices: filtered list of problem devices matching filterArg
|
||
* - apiBaseUrl: resolved base URL (for diagnostic logging in callers)
|
||
*/
|
||
async function fetchOfflineDevices(filterArg = '') {
|
||
const apiBaseUrl = process.env.APPSPACE_API_BASE_URL || process.env.APPSPACE_INSTANCE_URL || 'https://api.cloud.appspace.com';
|
||
const accessToken = await getValidAccessToken();
|
||
|
||
const response = await axios.get(`${apiBaseUrl}/api/v3/devices`, {
|
||
headers: { Authorization: `Bearer ${accessToken}` },
|
||
params: {
|
||
limit: 500,
|
||
healthStatus: 'Offline,LostCommunication,Failed',
|
||
status: 'Offline,LostCommunication,Failed'
|
||
},
|
||
timeout: 15000
|
||
});
|
||
|
||
const devices = response.data?.items || response.data || [];
|
||
let offlineDevices = devices.filter(isProblemDevice);
|
||
|
||
if (filterArg) {
|
||
const search = filterArg.toLowerCase().trim();
|
||
offlineDevices = offlineDevices.filter(d => {
|
||
const type = (d.deviceType || '').toLowerCase();
|
||
const name = (d.name || d.deviceName || '').toLowerCase();
|
||
return type.includes(search) || name.includes(search);
|
||
});
|
||
}
|
||
|
||
return { devices, offlineDevices, apiBaseUrl };
|
||
}
|
||
|
||
app.post('/webhook', async (req, res) => {
|
||
const payload = req.body;
|
||
|
||
const WEBHOOK_SECRET = process.env.WEBHOOK_SECRET;
|
||
if (WEBHOOK_SECRET) {
|
||
const receivedSecret = req.headers['x-webhook-secret'] || req.headers['x-secret'];
|
||
if (receivedSecret !== WEBHOOK_SECRET) {
|
||
logger.warn('Invalid secret header');
|
||
return res.status(401).send('Unauthorized');
|
||
}
|
||
}
|
||
|
||
const webhookType = payload.webhookType || '';
|
||
const data = payload.data || {};
|
||
|
||
// Log full payload only when debugging (can contain sensitive device/user data)
|
||
if (process.env.DEBUG_WEBHOOK === 'true' || process.env.NODE_ENV !== 'production') {
|
||
logger.debug('Appspace Webhook Received (full payload)', { payload });
|
||
} else {
|
||
logger.info('Appspace Webhook Received', { event: webhookType, device: data.deviceName || data.serialNumber || 'unknown' });
|
||
}
|
||
|
||
// ======================
|
||
// FILTER OUT PWA DEVICES
|
||
// ======================
|
||
if (data.deviceType === 'PWA' || data.deviceType?.toUpperCase() === 'PWA') {
|
||
if (isVerbose) logger.debug('Ignoring PWA device alert', { device: data.deviceName });
|
||
return res.status(200).send('Ignored PWA');
|
||
}
|
||
|
||
if (!webhookType.startsWith('DEVICE.HEALTHSTATUS.') && webhookType !== 'DEVICE.UNREGISTERED') {
|
||
if (isVerbose) logger.debug('Non-relevant event ignored');
|
||
return res.status(200).send('Ignored');
|
||
}
|
||
|
||
const eventKey = webhookType.replace('DEVICE.', '').replace('HEALTHSTATUS.', '').toUpperCase();
|
||
const isUnregistered = webhookType === 'DEVICE.UNREGISTERED';
|
||
const isProblem = isUnregistered || ['LOSTCOMMUNICATION', 'OFFLINE', 'FAILED'].includes(eventKey);
|
||
|
||
const accentColor = isProblem ? 'attention' : 'good';
|
||
const emoji = isUnregistered ? '🚫' : (isProblem ? '🔴' : '🟢');
|
||
|
||
const consoleBase = process.env.APPSPACE_CONSOLE_BASE_URL || 'https://app3.cloud.appspace.com';
|
||
const appspaceLink = `${consoleBase}/console/devices/details/overview?id=${data.deviceId}`;
|
||
|
||
// Enrich with Workspace ONE MDM (graceful; never blocks the alert)
|
||
const mdmDevice = await getMDMDeviceBySerial(data.serialNumber).catch(() => null);
|
||
const { mdmFacts, mdmConsoleLink } = buildMdmFactsAndLink(mdmDevice);
|
||
|
||
if (mdmDevice && mdmFacts.length > 0 && isVerbose) {
|
||
logger.debug('MDM data added', { device: data.deviceName });
|
||
}
|
||
|
||
const adaptiveCard = buildDeviceAlertCard({
|
||
eventKey,
|
||
data,
|
||
accentColor,
|
||
emoji,
|
||
appspaceLink,
|
||
mdmFacts,
|
||
mdmConsoleLink
|
||
});
|
||
|
||
try {
|
||
await axios.post('https://webexapis.com/v1/messages', {
|
||
roomId: process.env.WEBEX_ROOM_ID,
|
||
text: `${eventKey} on ${data.deviceName}`,
|
||
attachments: [{ contentType: "application/vnd.microsoft.card.adaptive", content: adaptiveCard }]
|
||
}, {
|
||
headers: { Authorization: `Bearer ${process.env.WEBEX_BOT_TOKEN}`, 'Content-Type': 'application/json' }
|
||
});
|
||
logger.info(`${eventKey} enriched card sent to Webex`);
|
||
} catch (err) {
|
||
logger.error('Failed to send card', { error: err.response?.data || err.message });
|
||
}
|
||
|
||
res.status(200).send('OK');
|
||
});
|
||
|
||
// ======================
|
||
// WEBEX BOT FRAMEWORK SETUP — WEBSOCKET MODE
|
||
// ======================
|
||
// Using WebSocket mode (no webhookUrl) to avoid public endpoint + rate-limit issues on restarts.
|
||
// Skipped in smoke tests (dummy token would cause noisy unhandled rejections; we only need the HTTP server + health for the smoke).
|
||
if (process.env.SMOKE_TEST !== 'true') {
|
||
framework = new Framework({
|
||
token: process.env.WEBEX_BOT_TOKEN,
|
||
// On startup, delete any stale WDM device registrations left behind by
|
||
// previously-dead-and-restarted instances. Prevents "excessive device
|
||
// registrations" errors that accumulate over the lifetime of the bot
|
||
// account when the process silently loses its Mercury socket and gets
|
||
// restarted. Safe for single-instance deployments; if you ever run
|
||
// multiple instances against the same bot token, revisit this.
|
||
removeDeviceRegistrationsOnStart: true
|
||
});
|
||
|
||
// If the framework fails to start (bad token, WebSocket handshake failure,
|
||
// Webex-side outage) the rejection would otherwise be swallowed by the global
|
||
// unhandledRejection handler and the bot would silently stay dead while
|
||
// Express keeps serving. Surface it explicitly so the failure is obvious.
|
||
/**
|
||
* Safely send a bot reply. A failed bot.say() (Mercury dead, rate limit, etc.)
|
||
* must not become an unhandled rejection — that leaves the framework wedged.
|
||
*/
|
||
async function safeSay(bot, payload) {
|
||
try {
|
||
await bot.say(payload);
|
||
} catch (err) {
|
||
logger.error('bot.say failed — Mercury may be disconnected', err);
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Restart the Webex framework in-process (stop → start) to recover a dead
|
||
* Mercury WebSocket without waiting for a full container restart.
|
||
*/
|
||
async function attemptBotReconnect(reason) {
|
||
if (reconnectInProgress || botHealth.exitingBecauseDead) return false;
|
||
|
||
reconnectInProgress = true;
|
||
botHealth.lastReconnectAttemptAt = new Date().toISOString();
|
||
botHealth.reconnectAttempts++;
|
||
|
||
logger.warn('Attempting in-process Webex framework restart', {
|
||
reason,
|
||
attempt: botHealth.reconnectAttempts
|
||
});
|
||
|
||
try {
|
||
botHealth.frameworkInitialized = false;
|
||
await framework.stop();
|
||
await framework.start();
|
||
// 'initialized' handler re-seeds mercury state and re-wires listeners.
|
||
logger.info('Webex framework restart completed', { reason });
|
||
botHealth.consecutiveFailures = 0;
|
||
return true;
|
||
} catch (err) {
|
||
logger.error('Webex framework restart failed', { reason, error: err });
|
||
return false;
|
||
} finally {
|
||
reconnectInProgress = false;
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Subscribe to Mercury lifecycle events so disconnects are visible in logs
|
||
* and permanent failures trigger an immediate reconnect attempt.
|
||
*/
|
||
function wireMercuryListeners() {
|
||
const mercury = framework?.webex?.internal?.mercury;
|
||
if (!mercury || typeof mercury.on !== 'function') return;
|
||
|
||
mercury.on('offline', () => {
|
||
botHealth.mercuryConnected = false;
|
||
logger.warn('Mercury offline event received');
|
||
});
|
||
|
||
mercury.on('online', () => {
|
||
botHealth.mercuryConnected = true;
|
||
botHealth.consecutiveFailures = 0;
|
||
botHealth.lastHealthyAt = new Date().toISOString();
|
||
logger.info('Mercury online event received');
|
||
});
|
||
|
||
mercury.on('offline.transient', () => {
|
||
logger.warn('Mercury transient offline — SDK should auto-reconnect');
|
||
});
|
||
|
||
mercury.on('offline.permanent', () => {
|
||
logger.error('Mercury permanent offline — triggering framework restart');
|
||
attemptBotReconnect('mercury.offline.permanent');
|
||
});
|
||
}
|
||
|
||
framework.start().catch((err) => {
|
||
logger.error('Webex framework failed to start — exiting so Docker can restart', err);
|
||
// Without this, webhooks keep working but commands never will until manual restart.
|
||
setTimeout(() => shutdown(true), 2000).unref();
|
||
});
|
||
|
||
framework.on('initialized', () => {
|
||
logger.info('Webex Bot Framework initialized (WebSocket mode)');
|
||
botHealth.frameworkInitialized = true;
|
||
// Seed the watchdog state so /health has something meaningful to report
|
||
// before the first watchdog tick fires.
|
||
const mercury = framework.webex?.internal?.mercury;
|
||
if (mercury) {
|
||
botHealth.mercuryConnected = !!mercury.connected;
|
||
botHealth.lastHealthyAt = new Date().toISOString();
|
||
}
|
||
wireMercuryListeners();
|
||
});
|
||
|
||
framework.on('spawn', (bot, id, addedBy) => {
|
||
if (isVerbose) {
|
||
if (addedBy) {
|
||
logger.debug('Bot added to new space', { addedBy });
|
||
} else {
|
||
logger.debug('Bot loaded space', { title: bot.room?.title || 'Unknown' });
|
||
}
|
||
}
|
||
});
|
||
|
||
// Forward the framework's internal 'log' events into our structured logger.
|
||
// Without this, framework-internal diagnostics (device registration issues,
|
||
// membership rule denials, etc.) are lost — which is a big part of why the
|
||
// Mercury silent-death is hard to diagnose from our logs alone.
|
||
framework.on('log', (msg) => {
|
||
logger.info('[framework]', { message: msg });
|
||
});
|
||
|
||
if (isVerbose) {
|
||
logger.debug('Webex Bot Framework starting in WebSocket mode...');
|
||
}
|
||
|
||
// ======================
|
||
// MERCURY WATCHDOG
|
||
// ======================
|
||
// The Webex SDK's Mercury WebSocket (server-push event stream) can die
|
||
// silently — network blip, WDM device TTL expiry, or a Cisco-side hiccup —
|
||
// without ever emitting an error the framework surfaces. When that happens
|
||
// the HTTP paths (webhooks, message-send API) keep working but the bot
|
||
// stops receiving commands. This watchdog polls the SDK's live
|
||
// `mercury.connected` boolean; if it stays false for enough consecutive
|
||
// checks, we log an error, mark ourselves unhealthy, and eventually exit
|
||
// so Docker's restart policy brings us back with a fresh connection.
|
||
const WATCHDOG_INTERVAL_MS = 60 * 1000; // check every minute
|
||
const WATCHDOG_WARN_AFTER = 1; // 1 consecutive miss → warn (~1 min)
|
||
const WATCHDOG_RECONNECT_AFTER = 2; // 2 consecutive misses → in-process restart (~2 min)
|
||
const WATCHDOG_EXIT_AFTER = 4; // 4 consecutive misses → exit (~4 min if restart failed)
|
||
|
||
const watchdogTimer = setInterval(() => {
|
||
botHealth.lastCheckAt = new Date().toISOString();
|
||
|
||
// If the framework hasn't finished initializing yet, don't count misses
|
||
// — we're still in the startup window.
|
||
if (!botHealth.frameworkInitialized || reconnectInProgress) {
|
||
return;
|
||
}
|
||
|
||
const mercury = framework.webex?.internal?.mercury;
|
||
const connected = !!(mercury && mercury.connected);
|
||
botHealth.mercuryConnected = connected;
|
||
|
||
if (connected) {
|
||
if (botHealth.consecutiveFailures > 0) {
|
||
logger.info('Mercury WebSocket recovered', {
|
||
afterFailures: botHealth.consecutiveFailures
|
||
});
|
||
}
|
||
botHealth.consecutiveFailures = 0;
|
||
botHealth.lastHealthyAt = botHealth.lastCheckAt;
|
||
return;
|
||
}
|
||
|
||
botHealth.consecutiveFailures++;
|
||
|
||
if (botHealth.consecutiveFailures === WATCHDOG_WARN_AFTER) {
|
||
logger.warn('Mercury WebSocket appears disconnected — bot may be silent', {
|
||
consecutiveFailures: botHealth.consecutiveFailures,
|
||
lastHealthyAt: botHealth.lastHealthyAt
|
||
});
|
||
} else if (botHealth.consecutiveFailures === WATCHDOG_RECONNECT_AFTER) {
|
||
attemptBotReconnect('watchdog');
|
||
} else if (botHealth.consecutiveFailures >= WATCHDOG_EXIT_AFTER && !botHealth.exitingBecauseDead) {
|
||
botHealth.exitingBecauseDead = true;
|
||
logger.error('Mercury WebSocket dead for sustained period — exiting so Docker restart policy can reconnect', {
|
||
consecutiveFailures: botHealth.consecutiveFailures,
|
||
lastHealthyAt: botHealth.lastHealthyAt,
|
||
reconnectAttempts: botHealth.reconnectAttempts
|
||
});
|
||
clearInterval(watchdogTimer);
|
||
// Give the log line + any in-flight HTTP responses a beat to flush,
|
||
// then trigger the same graceful shutdown path as SIGTERM. If that
|
||
// hangs, shutdown()'s own safety timeout forces exit.
|
||
setTimeout(() => shutdown(), 500).unref();
|
||
}
|
||
}, WATCHDOG_INTERVAL_MS);
|
||
// .unref() so this timer doesn't hold the process open by itself.
|
||
watchdogTimer.unref();
|
||
|
||
/**
|
||
* Extract the trailing filter portion of a command message, regardless of whether
|
||
* the message came from a DM (`offline ios`) or a group-space mention
|
||
* (`@appspace offline ios`).
|
||
*
|
||
* The framework sets `trigger.args = trigger.text.split(' ')`, and in group-space
|
||
* mentions `trigger.text` still has the bot's display name at the start. So we can't
|
||
* just `slice(1)` — that would take everything after the display name (including
|
||
* the command word itself) and treat it as the filter.
|
||
*
|
||
* Strategy: find the command word (case-insensitively) in the args array, then
|
||
* everything after it is the filter.
|
||
*/
|
||
function extractFilterArg(trigger, commandWord) {
|
||
const args = trigger.args || [];
|
||
const idx = args.findIndex(a => (a || '').toLowerCase() === commandWord.toLowerCase());
|
||
if (idx === -1 || idx === args.length - 1) return '';
|
||
return args.slice(idx + 1).join(' ').toLowerCase().trim();
|
||
}
|
||
|
||
// NOTE: keep as a string phrase (NOT a regex). The framework compiles string
|
||
// phrases into `/(^| )<phrase>($| )/i`, which:
|
||
// - correctly matches after the bot's display name in group-space mentions
|
||
// (e.g. `@appspace offline ios` → trigger.text = "Appspace offline ios")
|
||
// - uses SPACE delimiters (not `\b`), so it will NOT match the substring
|
||
// "offline" inside "restart-offline" (preceded by `-`, not space).
|
||
// A regex like /^offline\b/i is tested directly against trigger.text and would
|
||
// fail on any mentioned message because the display name comes first.
|
||
framework.hears('offline', async (bot, trigger) => {
|
||
try {
|
||
if (isVerbose) {
|
||
logger.debug('offline command received', { user: trigger.person?.displayName || 'Unknown' });
|
||
}
|
||
|
||
const filterArg = extractFilterArg(trigger, 'offline');
|
||
|
||
await safeSay(bot, { markdown: '🔍 Querying current offline / lost devices from Appspace...' });
|
||
|
||
// Pre-resolve for diagnostic logging in the catch block.
|
||
let apiBaseUrl = process.env.APPSPACE_API_BASE_URL || process.env.APPSPACE_INSTANCE_URL || 'https://api.cloud.appspace.com';
|
||
|
||
try {
|
||
const fetched = await fetchOfflineDevices(filterArg);
|
||
const { devices, offlineDevices } = fetched;
|
||
apiBaseUrl = fetched.apiBaseUrl;
|
||
|
||
if (offlineDevices.length === 0) {
|
||
const msg = filterArg
|
||
? `✅ No **${filterArg}** devices are currently offline.`
|
||
: '✅ All devices are currently online or in sync.';
|
||
return safeSay(bot, { markdown: msg });
|
||
}
|
||
|
||
const consoleBase = process.env.APPSPACE_CONSOLE_BASE_URL || 'https://app3.cloud.appspace.com';
|
||
const consoleUrl = `${consoleBase}/console/devices`;
|
||
|
||
// Enrich the devices we will display (cap at 30) with MDM + per-device Appspace links.
|
||
// Using a rich Markdown list (instead of table) so everything is consolidated per device
|
||
// and links render properly as clickable items.
|
||
const MAX_ROWS = 30;
|
||
const displayDevices = offlineDevices.slice(0, MAX_ROWS);
|
||
const enriched = await Promise.all(displayDevices.map(async (d) => {
|
||
const mdmDevice = await getMDMDeviceBySerial(d.serialNumber).catch(() => null);
|
||
let mdmInfo = '';
|
||
let mdmConsoleLink = null;
|
||
if (mdmDevice) {
|
||
const { mdmFacts, mdmConsoleLink: link } = buildMdmFactsAndLink(mdmDevice);
|
||
mdmConsoleLink = link;
|
||
if (mdmFacts.length > 0) {
|
||
mdmInfo = mdmFacts.map(f => `${f.title}: ${f.value}`).join(' | ');
|
||
}
|
||
}
|
||
const devId = d.deviceId || d.id || '';
|
||
const appspaceLink = devId ? `${consoleBase}/console/devices/details/overview?id=${devId}` : null;
|
||
|
||
return {
|
||
...d,
|
||
mdmInfo,
|
||
mdmConsoleLink,
|
||
appspaceLink
|
||
};
|
||
}));
|
||
|
||
// Group by location for better readability. Each device gets:
|
||
// - Name + basic Appspace info on first line (under bullet)
|
||
// - MDM info on second indented line
|
||
// - Links on third indented line
|
||
// No deep nesting; only bullet the device line, indent the rest.
|
||
const grouped = {};
|
||
enriched.forEach(e => {
|
||
const loc = e.locationName || 'Unknown Location';
|
||
if (!grouped[loc]) grouped[loc] = [];
|
||
grouped[loc].push(e);
|
||
});
|
||
|
||
// Webex caps a single message at 7439 characters before encryption.
|
||
// Build the body incrementally and stop appending devices once we approach the
|
||
// budget, leaving headroom for the header + console-link footer + a truncation note.
|
||
// The user can always click through to the full console list.
|
||
const WEBEX_BODY_BUDGET = 6500;
|
||
const filterNote = filterArg ? ` (filtered: ${filterArg})` : '';
|
||
const limitNote = devices.length >= 500 ? ' (results may be truncated — see console for full list)' : '';
|
||
|
||
let devicesList = '';
|
||
let renderedCount = 0;
|
||
let truncated = false;
|
||
|
||
outer: for (const loc of Object.keys(grouped).sort()) {
|
||
const locHeader = `**${loc}**\n`;
|
||
// If even the location header won't fit, stop entirely.
|
||
if (devicesList.length + locHeader.length > WEBEX_BODY_BUDGET) {
|
||
truncated = true;
|
||
break;
|
||
}
|
||
devicesList += locHeader;
|
||
|
||
for (const e of grouped[loc]) {
|
||
const name = (e.name || e.deviceName || 'Unknown');
|
||
const status = getDeviceHealthStatus(e) || 'Unknown';
|
||
const ip = e.ipAddress || 'N/A';
|
||
const type = e.deviceType || '—';
|
||
|
||
let entry = `- **${name}** (${status}) - IP: ${ip} Type: ${type}\n`;
|
||
entry += e.mdmInfo ? ` MDM: ${e.mdmInfo}\n` : ` No MDM data\n`;
|
||
let linksLine = '';
|
||
if (e.appspaceLink) linksLine += `[🔗 Appspace Console](${e.appspaceLink})`;
|
||
if (e.mdmConsoleLink) {
|
||
if (linksLine) linksLine += ' ';
|
||
linksLine += `[🔗 Workspace ONE Console](${e.mdmConsoleLink})`;
|
||
}
|
||
if (linksLine) entry += ` ${linksLine}\n`;
|
||
entry += '\n';
|
||
|
||
if (devicesList.length + entry.length > WEBEX_BODY_BUDGET) {
|
||
truncated = true;
|
||
break outer;
|
||
}
|
||
|
||
devicesList += entry;
|
||
renderedCount++;
|
||
}
|
||
}
|
||
|
||
// Truncation note covers both (a) enrichment cap (MAX_ROWS < total offline)
|
||
// and (b) character-budget truncation that stopped us mid-render.
|
||
const notShown = offlineDevices.length - renderedCount;
|
||
if (notShown > 0 || truncated) {
|
||
devicesList += `... (${notShown} more not shown, see full list in Appspace Console)\n`;
|
||
}
|
||
|
||
const message = `**📊 Offline Devices Snapshot (${offlineDevices.length})${filterNote}${limitNote}**\n\n` +
|
||
devicesList +
|
||
`[🔗 Open Devices in Appspace Console](${consoleUrl})`;
|
||
|
||
if (isVerbose) {
|
||
logger.debug(`Offline query returned ${offlineDevices.length} matching devices`);
|
||
}
|
||
|
||
await safeSay(bot, { markdown: message });
|
||
|
||
} catch (err) {
|
||
const attemptedUrl = `${apiBaseUrl}/api/v3/devices`;
|
||
logger.error('Offline query failed', {
|
||
attemptedUrl,
|
||
error: err.message,
|
||
code: err.code
|
||
});
|
||
await safeSay(bot, { markdown: `⚠️ Failed to query offline devices.\n\n${err.message || 'Unknown error'}` });
|
||
}
|
||
} catch (err) {
|
||
logger.error('offline command handler failed unexpectedly', err);
|
||
await safeSay(bot, { markdown: '⚠️ An unexpected error occurred while processing the offline command.' });
|
||
}
|
||
});
|
||
|
||
// restart-offline [filter]
|
||
// For every currently-offline (Offline / LostCommunication / Failed) Appspace device
|
||
// whose serial maps to a Workspace ONE record, send a SoftReset (reboot) command.
|
||
// Hard-capped at MAX_RESTART_BATCH per invocation; use a filter to narrow further.
|
||
// Always re-queries Appspace at execution time, so devices that came back online
|
||
// since the user last looked will NOT be rebooted.
|
||
// See note on the 'offline' handler above: string phrases handle mentions
|
||
// correctly (framework wraps them as `(^| )restart-offline($| )/i`).
|
||
framework.hears('restart-offline', async (bot, trigger) => {
|
||
try {
|
||
const MAX_RESTART_BATCH = 50;
|
||
const CONCURRENCY = 3;
|
||
|
||
const filterArg = extractFilterArg(trigger, 'restart-offline');
|
||
|
||
const userLabel = trigger.person?.emails?.[0] || trigger.person?.displayName || 'Unknown';
|
||
logger.info('restart-offline invoked', { user: userLabel, filter: filterArg || '(none)' });
|
||
|
||
await safeSay(bot, { markdown: `🔁 Querying current offline devices${filterArg ? ` matching \`${filterArg}\`` : ''} from Appspace...` });
|
||
|
||
let offlineDevices;
|
||
try {
|
||
const fetched = await fetchOfflineDevices(filterArg);
|
||
offlineDevices = fetched.offlineDevices;
|
||
} catch (err) {
|
||
logger.error('restart-offline: Appspace query failed', err);
|
||
return safeSay(bot, { markdown: `⚠️ Failed to query offline devices from Appspace.\n\n${err.message || 'Unknown error'}` });
|
||
}
|
||
|
||
if (offlineDevices.length === 0) {
|
||
return safeSay(bot, { markdown: filterArg
|
||
? `✅ No **${filterArg}** devices are currently offline. Nothing to restart.`
|
||
: '✅ No devices are currently offline. Nothing to restart.' });
|
||
}
|
||
|
||
if (offlineDevices.length > MAX_RESTART_BATCH) {
|
||
return safeSay(bot, { markdown:
|
||
`🛑 **${offlineDevices.length}** offline devices match — that exceeds the safety cap of **${MAX_RESTART_BATCH}** per invocation.\n\n` +
|
||
`Please narrow with a filter (e.g. \`restart-offline ios\`, \`restart-offline windows\`, or part of a device name) and try again.`
|
||
});
|
||
}
|
||
|
||
// Resolve each offline device to its WS1 numeric Id (needed for the SoftReset command).
|
||
const candidates = await Promise.all(offlineDevices.map(async (d) => {
|
||
const mdmDevice = await getMDMDeviceBySerial(d.serialNumber).catch(() => null);
|
||
return {
|
||
name: d.name || d.deviceName || 'Unknown',
|
||
location: d.locationName || 'Unknown',
|
||
serial: d.serialNumber || '(no serial)',
|
||
mdmId: mdmDevice?.Id?.Value || null
|
||
};
|
||
}));
|
||
|
||
const withMdm = candidates.filter(c => c.mdmId);
|
||
const withoutMdm = candidates.filter(c => !c.mdmId);
|
||
|
||
if (withMdm.length === 0) {
|
||
return safeSay(bot, { markdown:
|
||
`⚠️ Found **${offlineDevices.length}** offline device(s), but none have a Workspace ONE record (matched by serial). Nothing to restart.`
|
||
});
|
||
}
|
||
|
||
await safeSay(bot, { markdown:
|
||
`🔁 Sending **SoftReset** to **${withMdm.length}** device(s) via Workspace ONE...` +
|
||
(withoutMdm.length > 0 ? `\n_(${withoutMdm.length} offline device(s) have no WS1 record — skipping those.)_` : '')
|
||
});
|
||
|
||
// Bounded concurrency so we don't fire 50 reboots at WS1 in parallel.
|
||
const results = [];
|
||
for (let i = 0; i < withMdm.length; i += CONCURRENCY) {
|
||
const batch = withMdm.slice(i, i + CONCURRENCY);
|
||
const batchResults = await Promise.all(batch.map(async (c) => {
|
||
const r = await sendMDMRebootCommand(c.mdmId);
|
||
return { ...c, ...r };
|
||
}));
|
||
results.push(...batchResults);
|
||
}
|
||
|
||
const successes = results.filter(r => r.success);
|
||
const failures = results.filter(r => !r.success);
|
||
|
||
logger.info('restart-offline executed', {
|
||
user: userLabel,
|
||
filter: filterArg || '(none)',
|
||
matched: offlineDevices.length,
|
||
withMdm: withMdm.length,
|
||
withoutMdm: withoutMdm.length,
|
||
succeeded: successes.length,
|
||
failed: failures.length
|
||
});
|
||
|
||
// Webex 7439-char limit applies here too. Keep the report compact.
|
||
const WEBEX_BODY_BUDGET = 6500;
|
||
let msg = `**🔁 Restart Complete**${filterArg ? ` _(filter: \`${filterArg}\`)_` : ''}\n\n` +
|
||
`- Offline matched: **${offlineDevices.length}**\n` +
|
||
`- Restart command sent: **${successes.length}**\n` +
|
||
(failures.length > 0 ? `- Failed: **${failures.length}**\n` : '') +
|
||
(withoutMdm.length > 0 ? `- Skipped (no WS1 record): **${withoutMdm.length}**\n` : '') +
|
||
`\n_Note: WS1 only queues the command; an iOS device must be Supervised for SoftReset to actually execute._\n`;
|
||
|
||
if (failures.length > 0) {
|
||
msg += `\n**Failures:**\n`;
|
||
let failureLines = '';
|
||
let shown = 0;
|
||
for (const f of failures) {
|
||
const line = `- ${f.name} (${f.serial}): ${f.error || `HTTP ${f.status || 'unknown'}`}\n`;
|
||
if (msg.length + failureLines.length + line.length > WEBEX_BODY_BUDGET) break;
|
||
failureLines += line;
|
||
shown++;
|
||
}
|
||
msg += failureLines;
|
||
if (shown < failures.length) {
|
||
msg += `- ... and ${failures.length - shown} more failure(s) (see service logs)\n`;
|
||
}
|
||
}
|
||
|
||
if (withoutMdm.length > 0 && msg.length < WEBEX_BODY_BUDGET - 500) {
|
||
msg += `\n**Skipped (no WS1 record):**\n`;
|
||
let skipLines = '';
|
||
let shown = 0;
|
||
for (const s of withoutMdm) {
|
||
const line = `- ${s.name} at ${s.location} (serial: ${s.serial})\n`;
|
||
if (msg.length + skipLines.length + line.length > WEBEX_BODY_BUDGET) break;
|
||
skipLines += line;
|
||
shown++;
|
||
}
|
||
msg += skipLines;
|
||
if (shown < withoutMdm.length) {
|
||
msg += `- ... and ${withoutMdm.length - shown} more (see service logs)\n`;
|
||
}
|
||
}
|
||
|
||
await safeSay(bot, { markdown: msg });
|
||
} catch (err) {
|
||
logger.error('restart-offline command handler failed unexpectedly', err);
|
||
await safeSay(bot, { markdown: '⚠️ An unexpected error occurred while processing restart-offline.' });
|
||
}
|
||
});
|
||
|
||
framework.hears('help', async (bot) => {
|
||
await safeSay(bot, { markdown:
|
||
'**Commands:**\n' +
|
||
'• `offline [filter]` — Current offline / Lost / Failed devices from Appspace (optional name/type filter, e.g. `offline ios`)\n' +
|
||
'• `restart-offline [filter]` — Send a SoftReset (reboot) via Workspace ONE to every currently offline device that has a WS1 record. Optional filter narrows the set. Capped at 50 per invocation.\n' +
|
||
'• `help` — This message\n\n' +
|
||
'See README for setup, Docker usage, and debug flags.'
|
||
});
|
||
});
|
||
}
|
||
|
||
server = app.listen(PORT, () => {
|
||
logger.info(`🚀 Server listening on port ${PORT}`);
|
||
logger.info(` Appspace webhook: POST /webhook`);
|
||
logger.info(` Webex bot: WebSocket mode (no public HTTP webhook registered)`);
|
||
if (process.env.PUBLIC_HOST) {
|
||
logger.info(` Public host: ${process.env.PUBLIC_HOST}`);
|
||
}
|
||
logger.info(' Press Ctrl+C or send SIGTERM for graceful shutdown (Docker-friendly)');
|
||
});
|
||
|
||
// Register signal handlers for graceful shutdown (critical for Docker and orchestrators)
|
||
process.on('SIGTERM', () => {
|
||
logger.info('📡 Received SIGTERM (Docker stop / orchestrator)');
|
||
shutdown();
|
||
});
|
||
|
||
process.on('SIGINT', () => {
|
||
logger.info('📡 Received SIGINT (Ctrl+C)');
|
||
shutdown();
|
||
});
|