require('dotenv').config(); const express = require('express'); const axios = require('axios'); const Framework = require('webex-node-bot-framework'); const app = express(); const PORT = process.env.PORT || 3000; // ====================== // STARTUP VALIDATION (fail fast on missing critical config) // ====================== const REQUIRED_ENV = [ 'WEBEX_BOT_TOKEN', 'WEBEX_ROOM_ID', 'APPSPACE_INSTANCE_URL', 'APPSPACE_SUBJECT_ID', 'APPSPACE_REFRESH_TOKEN', 'APPSPACE_API_BASE_URL' ]; const missing = REQUIRED_ENV.filter(k => !process.env[k]); if (missing.length > 0) { console.error('❌ Missing required environment variables:'); missing.forEach(k => console.error(` - ${k}`)); console.error(' See .env.example for details.'); process.exit(1); } // Logging configuration for Docker / production friendliness const isVerbose = process.env.DEBUG === 'true' || process.env.NODE_ENV !== 'production'; const useJsonLogs = process.env.LOG_FORMAT === 'json' || process.env.NODE_ENV === 'production'; /** * Minimal structured logger. * - Human readable (with emojis) in dev * - JSON lines when LOG_FORMAT=json or NODE_ENV=production (great for Docker log collectors) */ const logger = { info: (msg, meta = {}) => log('info', msg, meta), warn: (msg, meta = {}) => log('warn', msg, meta), error: (msg, meta = {}) => log('error', msg, meta), debug: (msg, meta = {}) => { if (isVerbose) log('debug', msg, meta); } }; // Normalize whatever was passed as `meta` into a plain object suitable for JSON // logging. Errors have no enumerable own properties, so JSON.stringify(err) ⇒ "{}". // We pull out the useful bits (name/message/stack + axios response shape) so failures // are actually visible in the log stream. function normalizeMeta(meta) { if (meta == null) return {}; if (meta instanceof Error) { const out = { error: meta.message, errorName: meta.name, stack: meta.stack }; if (meta.code) out.code = meta.code; if (meta.response) { out.responseStatus = meta.response.status; out.responseData = meta.response.data; } return out; } if (typeof meta !== 'object') return { value: meta }; return meta; } function log(level, msg, meta = {}) { const timestamp = new Date().toISOString(); const normMeta = normalizeMeta(meta); if (useJsonLogs) { const entry = { timestamp, level, msg, ...normMeta }; // Remove undefined Object.keys(entry).forEach(k => entry[k] === undefined && delete entry[k]); console.log(JSON.stringify(entry)); } else { const emoji = level === 'error' ? '❌' : level === 'warn' ? '⚠️' : level === 'debug' ? '🐛' : 'ℹ️'; const metaStr = Object.keys(normMeta).length ? ' ' + JSON.stringify(normMeta) : ''; const out = level === 'error' ? console.error : console.log; out(`${emoji} ${msg}${metaStr}`); } } // Will be assigned when the server starts listening let server; let framework; // Graceful shutdown function (used by signals and crash handlers) function shutdown(force = false) { logger.info('🛑 Graceful shutdown initiated...'); const exit = (code = 0) => { logger.info(`👋 Process exiting with code ${code}`); process.exit(code); }; // Stop the Webex framework (important for WebSocket mode to clean up device registration) if (typeof framework !== 'undefined' && framework.stop) { framework.stop().then(() => { logger.info('✅ Webex framework stopped'); if (server) { server.close(() => { logger.info('✅ HTTP server closed'); exit(0); }); } else { exit(0); } }).catch((err) => { logger.error('Error stopping framework:', err); if (server) server.close(() => exit(1)); else exit(1); }); } else if (server) { server.close(() => { logger.info('✅ HTTP server closed'); exit(0); }); } else { exit(0); } // ALWAYS arm a safety timeout. If any step above hangs (e.g. framework.stop() // never resolves), we still need to exit before Docker's 10s SIGKILL. When called // from a crash handler (`force=true`) the process is already unhealthy, so exit // faster to minimize the window where we're wedged. setTimeout(() => { logger.error('⏱️ Graceful shutdown timed out. Forcing exit.'); exit(1); }, force ? 3000 : 8000).unref(); } // Basic crash handlers for container environments (Docker/K8s will restart on non-zero exit) // Uncaught is fatal -> shutdown. // UnhandledRejection (e.g. from Webex framework with bad creds or transient issues) just log; don't kill the main server. process.on('uncaughtException', (err) => { logger.error('Uncaught Exception. Initiating shutdown...', err); shutdown(true); }); process.on('unhandledRejection', (reason) => { // `reason` is most often an Error but can be anything; the logger normalizes it. // If something rejected with a non-Error value, wrap it so we still get a stack-y view. const payload = reason instanceof Error ? reason : { reason: typeof reason === 'object' ? JSON.stringify(reason) : String(reason) }; logger.error('Unhandled Rejection (logged, continuing)...', payload); // Do not call shutdown - keep the HTTP server running (e.g. Webex bot errors shouldn't kill webhook path) }); app.use(express.json()); // ====================== // APPSPACE TOKEN MANAGEMENT (with cooldown + longer timeout) // ====================== let currentAccessToken = null; let tokenExpiresAt = 0; let lastRefreshAttempt = 0; let inFlightAppspaceTokenPromise = null; function invalidateAppspaceToken() { currentAccessToken = null; tokenExpiresAt = 0; } async function getValidAccessToken() { const now = Math.floor(Date.now() / 1000); // Use cached token if still valid (2-minute safety buffer) if (currentAccessToken && now < tokenExpiresAt - 120) { return currentAccessToken; } // Coalesce concurrent callers (e.g. burst webhooks / bot commands at startup) // so we never fire more than one refresh in parallel. if (inFlightAppspaceTokenPromise) return inFlightAppspaceTokenPromise; // Prevent hammering the endpoint (minimum 10 seconds between refresh attempts) if (now - lastRefreshAttempt < 10) { logger.debug('Token refresh attempted too recently — using last known token if available'); if (currentAccessToken) return currentAccessToken; } lastRefreshAttempt = now; const instanceUrl = process.env.APPSPACE_INSTANCE_URL; if (!instanceUrl) { throw new Error('APPSPACE_INSTANCE_URL is not set in .env'); } inFlightAppspaceTokenPromise = (async () => { logger.info('Refreshing Appspace access token...'); try { const response = await axios.post(`${instanceUrl}/api/v3/authorization/token`, { subjectType: "Application", subjectId: process.env.APPSPACE_SUBJECT_ID, grantType: "refreshToken", refreshToken: process.env.APPSPACE_REFRESH_TOKEN }, { headers: { 'Content-Type': 'application/json' }, timeout: 30000 // Increased to 30 seconds }); const data = response.data; currentAccessToken = data.accessToken; tokenExpiresAt = Math.floor(Date.now() / 1000) + (data.expiresIn || 3600); logger.info('Appspace access token refreshed successfully', { expiresIn: data.expiresIn || 3600 }); return currentAccessToken; } catch (err) { logger.error('Token refresh failed', { error: err.response?.data || err.message }); const status = err.response?.status; if (status === 401 || status === 403) { invalidateAppspaceToken(); } if (err.code === 'ECONNABORTED') { logger.warn('Request timed out. Appspace token endpoint may be rate-limited.'); } throw new Error('Could not obtain valid Appspace access token. Try again in 30-60 seconds.'); } })().finally(() => { inFlightAppspaceTokenPromise = null; }); return inFlightAppspaceTokenPromise; } // ====================== // UTILITIES // ====================== /** * Normalize and format MDM timestamps (LastSystemSampleTime, LastSeen, etc.) * Handles missing 'Z' suffix from some WS1/Appspace responses and formats in * Eastern Time (America/New_York, DST-aware — so it renders as EST or EDT * automatically depending on the date). */ function formatMdmTimestamp(ts) { if (!ts) return 'Unknown'; let timestamp = ts.toString().trim(); // Some backends omit Z on what is effectively UTC; append if it looks like it needs it if (!timestamp.endsWith('Z') && !timestamp.includes('+') && timestamp.includes('-')) { timestamp += 'Z'; } const date = new Date(timestamp); if (isNaN(date.getTime())) { return timestamp; } return date.toLocaleString('en-US', { timeZone: 'America/New_York', month: 'numeric', day: 'numeric', year: 'numeric', hour: 'numeric', minute: '2-digit', hour12: true }); } /** * Extract a consistent set of MDM facts + Workspace ONE console link from a device record. * Handles the quirky casing and nested Id.Value shape in WS1 responses. * IMPORTANT: Use WS1_CONSOLE_BASE_URL (not WS1_BASE_URL) for the link base, as the API hostname (as....awmdm.com) * is different from the console hostname (cn....awmdm.com) and will produce broken links. */ function buildMdmFactsAndLink(mdmDevice) { if (!mdmDevice) { return { mdmFacts: [], mdmConsoleLink: null }; } const model = mdmDevice.Model || mdmDevice.model || 'Unknown'; const os = mdmDevice.OperatingSystem || 'Unknown'; const compliance = mdmDevice.ComplianceStatus || mdmDevice.complianceStatus || 'Unknown'; const lastSampleTime = formatMdmTimestamp(mdmDevice.LastSystemSampleTime); const lastSeenDisplay = formatMdmTimestamp(mdmDevice.LastSeen); const deviceId = mdmDevice.Id?.Value || mdmDevice.Uuid || mdmDevice.id || ''; let mdmConsoleLink = null; if (deviceId) { // Always use the console hostname (e.g. cn1896.awmdm.com), not the API hostname (e.g. as1896.awmdm.com) // WS1_BASE_URL is the API server; WS1_CONSOLE_BASE_URL (or equivalent) is for the web UI links. const ws1Base = process.env.WS1_CONSOLE_BASE_URL || 'https://cn1896.awmdm.com'; mdmConsoleLink = `${ws1Base}/AirWatch/#/AirWatch/Device/Details/Summary/${deviceId}`; } const mdmFacts = []; if (model && model !== 'Unknown') mdmFacts.push({ "title": "Model", "value": model }); if (os && os !== 'Unknown') mdmFacts.push({ "title": "OS Version", "value": os }); if (compliance && compliance !== 'Unknown') mdmFacts.push({ "title": "Compliance", "value": compliance }); if (lastSampleTime && lastSampleTime !== 'Unknown') mdmFacts.push({ "title": "Last Sample (ET)", "value": lastSampleTime }); if (lastSeenDisplay && lastSeenDisplay !== 'Unknown') mdmFacts.push({ "title": "Last Seen (ET)", "value": lastSeenDisplay }); return { mdmFacts, mdmConsoleLink }; } /** * Construct the complete Adaptive Card payload for Webex. * Keeps presentation logic out of the request handler. * * Note: Webex currently supports a maximum of Adaptive Cards 1.3 * (1.4+ is in the engineering backlog with no ETA as of 2025/2026). * See: https://developer.webex.com/docs/buttons-and-cards * Schema explorer: https://adaptivecards.io/explorer/ */ function buildDeviceAlertCard({ eventKey, data, accentColor, emoji, appspaceLink, mdmFacts = [], mdmConsoleLink = null }) { const adaptiveCard = { "$schema": "http://adaptivecards.io/schemas/adaptive-card.json", "type": "AdaptiveCard", "version": "1.3", // Webex currently supports a maximum of Adaptive Cards 1.3 (1.4+ is in backlog with no ETA) "body": [ { "type": "Container", "style": accentColor, "bleed": true, "items": [ { "type": "TextBlock", "text": `${emoji} Appspace Device Alert`, "weight": "bolder", "size": "medium", "wrap": true } ] }, { "type": "FactSet", "facts": [ { "title": "Event", "value": eventKey }, { "title": "Device", "value": data.deviceName || 'Unknown' }, { "title": "Location", "value": data.locationName || 'Unknown' }, { "title": "Type", "value": data.deviceType || 'N/A' }, { "title": "IP", "value": data.ipAddress || 'N/A' }, { "title": "Serial", "value": data.serialNumber || 'N/A' } ], "spacing": "Small" } ], "actions": [ { "type": "Action.OpenUrl", "title": "🔗 Appspace Console", "url": appspaceLink } ] }; if (mdmFacts.length > 0) { adaptiveCard.body.push({ "type": "Container", "separator": true, "items": [ { "type": "TextBlock", "text": "**MDM Status**", "weight": "bolder", "size": "medium", "wrap": true, "color": "Accent", "spacing": "Small" }, { "type": "FactSet", "facts": mdmFacts } ] }); if (mdmConsoleLink) { adaptiveCard.actions.push({ "type": "Action.OpenUrl", "title": "🔗 Workspace ONE Console", "url": mdmConsoleLink }); } } // If no MDM data, we intentionally omit any warning block to keep the card compact and focused on the alert. return adaptiveCard; } /** * Normalize health/status field from Appspace device objects. */ function getDeviceHealthStatus(device) { if (!device) return ''; return (device.status || device.healthStatus || '').toString().toUpperCase().trim(); } /** * Returns true for devices that are offline/lost/failed. * Shared logic for bot queries (and potentially future webhook use). */ function isProblemDevice(device) { const status = getDeviceHealthStatus(device); return ['OFFLINE', 'LOSTCOMMUNICATION', 'FAILED'].includes(status); } // ====================== // HEALTHCHECK // ====================== app.get('/health', (req, res) => { res.status(200).json({ status: 'healthy', environment: process.env.NODE_ENV || 'production', timestamp: new Date().toISOString() }); }); // ====================== // APPSPACE WEBHOOK → Alerts (now includes UNREGISTERED) // ====================== const { getMDMDeviceBySerial, sendMDMRebootCommand } = require('./mdm'); /** * Query Appspace for current problem devices (Offline / LostCommunication / Failed), * optionally narrowed by a free-text filter that matches deviceType or name. * Shared between `offline` and `restart-offline` so they always agree on what * counts as offline. * * Returns { devices, offlineDevices, apiBaseUrl } where: * - devices: raw page returned by Appspace (used to detect 500-row truncation) * - offlineDevices: filtered list of problem devices matching filterArg * - apiBaseUrl: resolved base URL (for diagnostic logging in callers) */ async function fetchOfflineDevices(filterArg = '') { const apiBaseUrl = process.env.APPSPACE_API_BASE_URL || process.env.APPSPACE_INSTANCE_URL || 'https://api.cloud.appspace.com'; const accessToken = await getValidAccessToken(); const response = await axios.get(`${apiBaseUrl}/api/v3/devices`, { headers: { Authorization: `Bearer ${accessToken}` }, params: { limit: 500, healthStatus: 'Offline,LostCommunication,Failed', status: 'Offline,LostCommunication,Failed' }, timeout: 15000 }); const devices = response.data?.items || response.data || []; let offlineDevices = devices.filter(isProblemDevice); if (filterArg) { const search = filterArg.toLowerCase().trim(); offlineDevices = offlineDevices.filter(d => { const type = (d.deviceType || '').toLowerCase(); const name = (d.name || d.deviceName || '').toLowerCase(); return type.includes(search) || name.includes(search); }); } return { devices, offlineDevices, apiBaseUrl }; } app.post('/webhook', async (req, res) => { const payload = req.body; const WEBHOOK_SECRET = process.env.WEBHOOK_SECRET; if (WEBHOOK_SECRET) { const receivedSecret = req.headers['x-webhook-secret'] || req.headers['x-secret']; if (receivedSecret !== WEBHOOK_SECRET) { logger.warn('Invalid secret header'); return res.status(401).send('Unauthorized'); } } const webhookType = payload.webhookType || ''; const data = payload.data || {}; // Log full payload only when debugging (can contain sensitive device/user data) if (process.env.DEBUG_WEBHOOK === 'true' || process.env.NODE_ENV !== 'production') { logger.debug('Appspace Webhook Received (full payload)', { payload }); } else { logger.info('Appspace Webhook Received', { event: webhookType, device: data.deviceName || data.serialNumber || 'unknown' }); } // ====================== // FILTER OUT PWA DEVICES // ====================== if (data.deviceType === 'PWA' || data.deviceType?.toUpperCase() === 'PWA') { if (isVerbose) logger.debug('Ignoring PWA device alert', { device: data.deviceName }); return res.status(200).send('Ignored PWA'); } if (!webhookType.startsWith('DEVICE.HEALTHSTATUS.') && webhookType !== 'DEVICE.UNREGISTERED') { if (isVerbose) logger.debug('Non-relevant event ignored'); return res.status(200).send('Ignored'); } const eventKey = webhookType.replace('DEVICE.', '').replace('HEALTHSTATUS.', '').toUpperCase(); const isUnregistered = webhookType === 'DEVICE.UNREGISTERED'; const isProblem = isUnregistered || ['LOSTCOMMUNICATION', 'OFFLINE', 'FAILED'].includes(eventKey); const accentColor = isProblem ? 'attention' : 'good'; const emoji = isUnregistered ? '🚫' : (isProblem ? '🔴' : '🟢'); const consoleBase = process.env.APPSPACE_CONSOLE_BASE_URL || 'https://app3.cloud.appspace.com'; const appspaceLink = `${consoleBase}/console/devices/details/overview?id=${data.deviceId}`; // Enrich with Workspace ONE MDM (graceful; never blocks the alert) const mdmDevice = await getMDMDeviceBySerial(data.serialNumber).catch(() => null); const { mdmFacts, mdmConsoleLink } = buildMdmFactsAndLink(mdmDevice); if (mdmDevice && mdmFacts.length > 0 && isVerbose) { logger.debug('MDM data added', { device: data.deviceName }); } const adaptiveCard = buildDeviceAlertCard({ eventKey, data, accentColor, emoji, appspaceLink, mdmFacts, mdmConsoleLink }); try { await axios.post('https://webexapis.com/v1/messages', { roomId: process.env.WEBEX_ROOM_ID, text: `${eventKey} on ${data.deviceName}`, attachments: [{ contentType: "application/vnd.microsoft.card.adaptive", content: adaptiveCard }] }, { headers: { Authorization: `Bearer ${process.env.WEBEX_BOT_TOKEN}`, 'Content-Type': 'application/json' } }); logger.info(`${eventKey} enriched card sent to Webex`); } catch (err) { logger.error('Failed to send card', { error: err.response?.data || err.message }); } res.status(200).send('OK'); }); // ====================== // WEBEX BOT FRAMEWORK SETUP — WEBSOCKET MODE // ====================== // Using WebSocket mode (no webhookUrl) to avoid public endpoint + rate-limit issues on restarts. // Skipped in smoke tests (dummy token would cause noisy unhandled rejections; we only need the HTTP server + health for the smoke). if (process.env.SMOKE_TEST !== 'true') { framework = new Framework({ token: process.env.WEBEX_BOT_TOKEN, }); // If the framework fails to start (bad token, WebSocket handshake failure, // Webex-side outage) the rejection would otherwise be swallowed by the global // unhandledRejection handler and the bot would silently stay dead while // Express keeps serving. Surface it explicitly so the failure is obvious. framework.start().catch((err) => { logger.error('Webex framework failed to start (bot will be unresponsive; HTTP server continues)', err); }); framework.on('initialized', () => { logger.info('Webex Bot Framework initialized (WebSocket mode)'); }); framework.on('spawn', (bot, id, addedBy) => { if (isVerbose) { if (addedBy) { logger.debug('Bot added to new space', { addedBy }); } else { logger.debug('Bot loaded space', { title: bot.room?.title || 'Unknown' }); } } }); if (isVerbose) { logger.debug('Webex Bot Framework starting in WebSocket mode...'); } /** * Extract the trailing filter portion of a command message, regardless of whether * the message came from a DM (`offline ios`) or a group-space mention * (`@appspace offline ios`). * * The framework sets `trigger.args = trigger.text.split(' ')`, and in group-space * mentions `trigger.text` still has the bot's display name at the start. So we can't * just `slice(1)` — that would take everything after the display name (including * the command word itself) and treat it as the filter. * * Strategy: find the command word (case-insensitively) in the args array, then * everything after it is the filter. */ function extractFilterArg(trigger, commandWord) { const args = trigger.args || []; const idx = args.findIndex(a => (a || '').toLowerCase() === commandWord.toLowerCase()); if (idx === -1 || idx === args.length - 1) return ''; return args.slice(idx + 1).join(' ').toLowerCase().trim(); } // NOTE: keep as a string phrase (NOT a regex). The framework compiles string // phrases into `/(^| )($| )/i`, which: // - correctly matches after the bot's display name in group-space mentions // (e.g. `@appspace offline ios` → trigger.text = "Appspace offline ios") // - uses SPACE delimiters (not `\b`), so it will NOT match the substring // "offline" inside "restart-offline" (preceded by `-`, not space). // A regex like /^offline\b/i is tested directly against trigger.text and would // fail on any mentioned message because the display name comes first. framework.hears('offline', async (bot, trigger) => { if (isVerbose) { logger.debug('offline command received', { user: trigger.person?.displayName || 'Unknown' }); } const filterArg = extractFilterArg(trigger, 'offline'); await bot.say({ markdown: '🔍 Querying current offline / lost devices from Appspace...' }); // Pre-resolve for diagnostic logging in the catch block. let apiBaseUrl = process.env.APPSPACE_API_BASE_URL || process.env.APPSPACE_INSTANCE_URL || 'https://api.cloud.appspace.com'; try { const fetched = await fetchOfflineDevices(filterArg); const { devices, offlineDevices } = fetched; apiBaseUrl = fetched.apiBaseUrl; if (offlineDevices.length === 0) { const msg = filterArg ? `✅ No **${filterArg}** devices are currently offline.` : '✅ All devices are currently online or in sync.'; return bot.say({ markdown: msg }); } const consoleBase = process.env.APPSPACE_CONSOLE_BASE_URL || 'https://app3.cloud.appspace.com'; const consoleUrl = `${consoleBase}/console/devices`; // Enrich the devices we will display (cap at 30) with MDM + per-device Appspace links. // Using a rich Markdown list (instead of table) so everything is consolidated per device // and links render properly as clickable items. const MAX_ROWS = 30; const displayDevices = offlineDevices.slice(0, MAX_ROWS); const enriched = await Promise.all(displayDevices.map(async (d) => { const mdmDevice = await getMDMDeviceBySerial(d.serialNumber).catch(() => null); let mdmInfo = ''; let mdmConsoleLink = null; if (mdmDevice) { const { mdmFacts, mdmConsoleLink: link } = buildMdmFactsAndLink(mdmDevice); mdmConsoleLink = link; if (mdmFacts.length > 0) { mdmInfo = mdmFacts.map(f => `${f.title}: ${f.value}`).join(' | '); } } const devId = d.deviceId || d.id || ''; const appspaceLink = devId ? `${consoleBase}/console/devices/details/overview?id=${devId}` : null; return { ...d, mdmInfo, mdmConsoleLink, appspaceLink }; })); // Group by location for better readability. Each device gets: // - Name + basic Appspace info on first line (under bullet) // - MDM info on second indented line // - Links on third indented line // No deep nesting; only bullet the device line, indent the rest. const grouped = {}; enriched.forEach(e => { const loc = e.locationName || 'Unknown Location'; if (!grouped[loc]) grouped[loc] = []; grouped[loc].push(e); }); // Webex caps a single message at 7439 characters before encryption. // Build the body incrementally and stop appending devices once we approach the // budget, leaving headroom for the header + console-link footer + a truncation note. // The user can always click through to the full console list. const WEBEX_BODY_BUDGET = 6500; const filterNote = filterArg ? ` (filtered: ${filterArg})` : ''; const limitNote = devices.length >= 500 ? ' (results may be truncated — see console for full list)' : ''; let devicesList = ''; let renderedCount = 0; let truncated = false; outer: for (const loc of Object.keys(grouped).sort()) { const locHeader = `**${loc}**\n`; // If even the location header won't fit, stop entirely. if (devicesList.length + locHeader.length > WEBEX_BODY_BUDGET) { truncated = true; break; } devicesList += locHeader; for (const e of grouped[loc]) { const name = (e.name || e.deviceName || 'Unknown'); const status = getDeviceHealthStatus(e) || 'Unknown'; const ip = e.ipAddress || 'N/A'; const type = e.deviceType || '—'; let entry = `- **${name}** (${status}) - IP: ${ip} Type: ${type}\n`; entry += e.mdmInfo ? ` MDM: ${e.mdmInfo}\n` : ` No MDM data\n`; let linksLine = ''; if (e.appspaceLink) linksLine += `[🔗 Appspace Console](${e.appspaceLink})`; if (e.mdmConsoleLink) { if (linksLine) linksLine += ' '; linksLine += `[🔗 Workspace ONE Console](${e.mdmConsoleLink})`; } if (linksLine) entry += ` ${linksLine}\n`; entry += '\n'; if (devicesList.length + entry.length > WEBEX_BODY_BUDGET) { truncated = true; break outer; } devicesList += entry; renderedCount++; } } // Truncation note covers both (a) enrichment cap (MAX_ROWS < total offline) // and (b) character-budget truncation that stopped us mid-render. const notShown = offlineDevices.length - renderedCount; if (notShown > 0 || truncated) { devicesList += `... (${notShown} more not shown, see full list in Appspace Console)\n`; } const message = `**📊 Offline Devices Snapshot (${offlineDevices.length})${filterNote}${limitNote}**\n\n` + devicesList + `[🔗 Open Devices in Appspace Console](${consoleUrl})`; if (isVerbose) { logger.debug(`Offline query returned ${offlineDevices.length} matching devices`); } await bot.say({ markdown: message }); } catch (err) { const attemptedUrl = `${apiBaseUrl}/api/v3/devices`; logger.error('Offline query failed', { attemptedUrl, error: err.message, code: err.code }); bot.say({ markdown: `⚠️ Failed to query offline devices.\n\n${err.message || 'Unknown error'}` }); } }); // restart-offline [filter] // For every currently-offline (Offline / LostCommunication / Failed) Appspace device // whose serial maps to a Workspace ONE record, send a SoftReset (reboot) command. // Hard-capped at MAX_RESTART_BATCH per invocation; use a filter to narrow further. // Always re-queries Appspace at execution time, so devices that came back online // since the user last looked will NOT be rebooted. // See note on the 'offline' handler above: string phrases handle mentions // correctly (framework wraps them as `(^| )restart-offline($| )/i`). framework.hears('restart-offline', async (bot, trigger) => { const MAX_RESTART_BATCH = 50; const CONCURRENCY = 3; const filterArg = extractFilterArg(trigger, 'restart-offline'); const userLabel = trigger.person?.emails?.[0] || trigger.person?.displayName || 'Unknown'; logger.info('restart-offline invoked', { user: userLabel, filter: filterArg || '(none)' }); await bot.say({ markdown: `🔁 Querying current offline devices${filterArg ? ` matching \`${filterArg}\`` : ''} from Appspace...` }); let offlineDevices; try { const fetched = await fetchOfflineDevices(filterArg); offlineDevices = fetched.offlineDevices; } catch (err) { logger.error('restart-offline: Appspace query failed', err); return bot.say({ markdown: `⚠️ Failed to query offline devices from Appspace.\n\n${err.message || 'Unknown error'}` }); } if (offlineDevices.length === 0) { return bot.say({ markdown: filterArg ? `✅ No **${filterArg}** devices are currently offline. Nothing to restart.` : '✅ No devices are currently offline. Nothing to restart.' }); } if (offlineDevices.length > MAX_RESTART_BATCH) { return bot.say({ markdown: `🛑 **${offlineDevices.length}** offline devices match — that exceeds the safety cap of **${MAX_RESTART_BATCH}** per invocation.\n\n` + `Please narrow with a filter (e.g. \`restart-offline ios\`, \`restart-offline windows\`, or part of a device name) and try again.` }); } // Resolve each offline device to its WS1 numeric Id (needed for the SoftReset command). const candidates = await Promise.all(offlineDevices.map(async (d) => { const mdmDevice = await getMDMDeviceBySerial(d.serialNumber).catch(() => null); return { name: d.name || d.deviceName || 'Unknown', location: d.locationName || 'Unknown', serial: d.serialNumber || '(no serial)', mdmId: mdmDevice?.Id?.Value || null }; })); const withMdm = candidates.filter(c => c.mdmId); const withoutMdm = candidates.filter(c => !c.mdmId); if (withMdm.length === 0) { return bot.say({ markdown: `⚠️ Found **${offlineDevices.length}** offline device(s), but none have a Workspace ONE record (matched by serial). Nothing to restart.` }); } await bot.say({ markdown: `🔁 Sending **SoftReset** to **${withMdm.length}** device(s) via Workspace ONE...` + (withoutMdm.length > 0 ? `\n_(${withoutMdm.length} offline device(s) have no WS1 record — skipping those.)_` : '') }); // Bounded concurrency so we don't fire 50 reboots at WS1 in parallel. const results = []; for (let i = 0; i < withMdm.length; i += CONCURRENCY) { const batch = withMdm.slice(i, i + CONCURRENCY); const batchResults = await Promise.all(batch.map(async (c) => { const r = await sendMDMRebootCommand(c.mdmId); return { ...c, ...r }; })); results.push(...batchResults); } const successes = results.filter(r => r.success); const failures = results.filter(r => !r.success); logger.info('restart-offline executed', { user: userLabel, filter: filterArg || '(none)', matched: offlineDevices.length, withMdm: withMdm.length, withoutMdm: withoutMdm.length, succeeded: successes.length, failed: failures.length }); // Webex 7439-char limit applies here too. Keep the report compact. const WEBEX_BODY_BUDGET = 6500; let msg = `**🔁 Restart Complete**${filterArg ? ` _(filter: \`${filterArg}\`)_` : ''}\n\n` + `- Offline matched: **${offlineDevices.length}**\n` + `- Restart command sent: **${successes.length}**\n` + (failures.length > 0 ? `- Failed: **${failures.length}**\n` : '') + (withoutMdm.length > 0 ? `- Skipped (no WS1 record): **${withoutMdm.length}**\n` : '') + `\n_Note: WS1 only queues the command; an iOS device must be Supervised for SoftReset to actually execute._\n`; if (failures.length > 0) { msg += `\n**Failures:**\n`; let failureLines = ''; let shown = 0; for (const f of failures) { const line = `- ${f.name} (${f.serial}): ${f.error || `HTTP ${f.status || 'unknown'}`}\n`; if (msg.length + failureLines.length + line.length > WEBEX_BODY_BUDGET) break; failureLines += line; shown++; } msg += failureLines; if (shown < failures.length) { msg += `- ... and ${failures.length - shown} more failure(s) (see service logs)\n`; } } if (withoutMdm.length > 0 && msg.length < WEBEX_BODY_BUDGET - 500) { msg += `\n**Skipped (no WS1 record):**\n`; let skipLines = ''; let shown = 0; for (const s of withoutMdm) { const line = `- ${s.name} at ${s.location} (serial: ${s.serial})\n`; if (msg.length + skipLines.length + line.length > WEBEX_BODY_BUDGET) break; skipLines += line; shown++; } msg += skipLines; if (shown < withoutMdm.length) { msg += `- ... and ${withoutMdm.length - shown} more (see service logs)\n`; } } await bot.say({ markdown: msg }); }); framework.hears('help', (bot) => { bot.say({ markdown: '**Commands:**\n' + '• `offline [filter]` — Current offline / Lost / Failed devices from Appspace (optional name/type filter, e.g. `offline ios`)\n' + '• `restart-offline [filter]` — Send a SoftReset (reboot) via Workspace ONE to every currently offline device that has a WS1 record. Optional filter narrows the set. Capped at 50 per invocation.\n' + '• `help` — This message\n\n' + 'See README for setup, Docker usage, and debug flags.' }); }); } server = app.listen(PORT, () => { logger.info(`🚀 Server listening on port ${PORT}`); logger.info(` Appspace webhook: POST /webhook`); logger.info(` Webex bot: WebSocket mode (no public HTTP webhook registered)`); if (process.env.PUBLIC_HOST) { logger.info(` Public host: ${process.env.PUBLIC_HOST}`); } logger.info(' Press Ctrl+C or send SIGTERM for graceful shutdown (Docker-friendly)'); }); // Register signal handlers for graceful shutdown (critical for Docker and orchestrators) process.on('SIGTERM', () => { logger.info('📡 Received SIGTERM (Docker stop / orchestrator)'); shutdown(); }); process.on('SIGINT', () => { logger.info('📡 Received SIGINT (Ctrl+C)'); shutdown(); });