// services/vcProvisionService.js // Supports /provision-vc (renamed from /vcprovision) and multiple organizations via optional org param. import { exec } from 'node:child_process'; import { promisify } from 'node:util'; import fs from 'node:fs'; import axios from 'axios'; import digicert from '../integrations/digicert/DigiCertClient.js'; import webex from '../integrations/webex/WebexClient.js'; import { logger } from '../utils/logger.js'; const execAsync = promisify(exec); const ROOT_CERTS_PATH = './storage/aeoroots.cer'; /** * Resolve org identifier (partial name or ID) to full org ID. * Fetches from /organizations if needed. */ async function resolveOrgId(identifier) { if (!identifier) return null; // If it looks like a full org ID, use as-is if (identifier.startsWith('Y2lzY29zcGFyazovL3VzL09SR0FOSVpBVElPTi8')) { return identifier; } logger('vc-provision', `Resolving org identifier: ${identifier}`); const orgsResponse = await webex.request('GET', 'organizations'); const orgs = orgsResponse.items || []; const lowerId = identifier.toLowerCase(); const match = orgs.find(org => org.id === identifier || org.displayName.toLowerCase().includes(lowerId) ); if (!match) { const available = orgs.map(o => `${o.displayName} (${o.id})`).join(', '); throw new Error(`Organization not found for "${identifier}". Available: ${available}`); } logger('vc-provision', `Resolved org: ${match.displayName} (${match.id})`); return match.id; } export async function provisionVideoDevice(bot, serialNumber, orgIdentifier = null) { if (!serialNumber) throw new Error('Serial number is required'); const deviceName = serialNumber; logger('vc-provision', `Starting full provisioning for ${deviceName}`); try { await bot.say('markdown', `🚀 Starting provisioning for device **${deviceName}**...\nThis may take 3–6 minutes.`); // Resolve orgId if orgIdentifier provided (name or id) let orgId = null; if (orgIdentifier) { orgId = await resolveOrgId(orgIdentifier); await bot.say('markdown', `📍 Using organization: ${orgId}`); } // 1. Find device await bot.say('markdown', '🔍 Finding device in Webex...'); const params = { serial: serialNumber }; if (orgId) params.orgId = orgId; const devicesResponse = await webex.request('GET', 'devices', null, params); if (!devicesResponse.items || devicesResponse.items.length === 0) { throw new Error(`No device found with serial ${serialNumber}`); } if (devicesResponse.items.length > 1) { throw new Error(`Multiple devices found with serial ${serialNumber}`); } const device = devicesResponse.items[0]; logger('vc-provision', `Found device: ${device.displayName || device.serial}`); await bot.say('markdown', `✅ Device found: **${device.displayName || device.serial}**`); // 2. Get location await bot.say('markdown', '📍 Retrieving location details...'); const location = await webex.request('GET', `locations/${device.locationId}`); // 3. Apply standard configuration FIRST await bot.say('markdown', '⚙️ Applying standard configuration...'); await applyStandardConfiguration(bot, device.id); // 4. Generate Private Key + CSR await bot.say('markdown', '🔑 Generating private key and CSR...'); const { privateKeyPem, csrPem, commonName } = await generateCSR(device, location); const csrBase64 = csrPem .replace(/-----BEGIN CERTIFICATE REQUEST-----/g, '') .replace(/-----END CERTIFICATE REQUEST-----/g, '') .replace(/\r?\n/g, '') .trim(); // 5. Enroll with DigiCert await bot.say('markdown', '📡 Submitting CSR to DigiCert...'); const enrollResponse = await digicert.enrollCertificate(csrBase64, commonName, device.ip || '0.0.0.0'); const requestId = enrollResponse.request_id; if (!requestId) throw new Error('No request_id returned from DigiCert'); // 6. Poll for certificate await bot.say('markdown', '⏳ Waiting for certificate from DigiCert...'); const certPem = await pollAndPickupCertificate(bot, requestId); // 7. Apply certificates await bot.say('markdown', '📥 Installing certificate...'); await applyCertificatesToDevice(bot, device.id, privateKeyPem, certPem); // 8. Add backdoor admin account await bot.say('markdown', '🔑 Adding backdoor admin account (`monitor`)...'); await addBackdoorAdmin(bot, device.id); // 9. Final reboot await bot.say('markdown', '🔄 Rebooting device...'); await webex.request('POST', `xapi/command/SystemUnit.Boot`, { deviceId: device.id, arguments: { Action: 'Restart', Force: 'True' } }); await bot.say('markdown', `🎉 **Provisioning completed successfully!**\n\n` + `**Device:** ${device.displayName || device.serial}\n` + `**Common Name:** ${commonName}\n\n` + `• Certificate installed and activated\n` + `• Standard configuration applied\n` + `• Backdoor admin account added\n` + `• Device rebooted` ); logger('vc-provision', `✅ Full provisioning completed for ${deviceName}`); } catch (error) { logger('vc-provision', `Provisioning failed: ${error.message}`, 'error'); await bot.say('markdown', `❌ **Provisioning failed**\n\n${error.message}`); throw error; } } // ==================== STANDARD CONFIGURATION WITH 429 HANDLING ==================== async function applyStandardConfiguration(bot, deviceId) { const configSettings = [ { path: "HttpClient.Mode", value: "On" }, { path: "HttpClient.UseHttpProxy", value: "Off" }, { path: "NetworkServices.Websocket", value: "FollowHTTPService" }, { path: "Proximity.Services.CallControl", value: "Enabled" }, { path: "RoomAnalytics.AmbientNoiseEstimation.Mode", value: "On" }, { path: "RoomAnalytics.PeopleCountOutOfCall", value: "On" }, { path: "RoomAnalytics.PeoplePresenceDetector", value: "On" }, { path: "RoomAnalytics.ReverberationTime.Mode", value: "On" }, { path: "Standby.Delay", value: 10 }, { path: "Standby.Signage.Mode", value: "Off" }, { path: "Standby.Signage.Url", value: "https://app.onfirstup.com/embed/9f3b4f1b-4acc-4076-aafe-42d09f3ed384" }, { path: "Time.DateFormat", value: "MM_DD_YY" }, { path: "Time.TimeFormat", value: "12H" }, { path: "UserInterface.Features.Call.JoinGoogleMeet", value: "Auto" }, { path: "UserInterface.Features.Call.JoinMicrosoftTeamsDirectGuestJoin", value: "Auto" }, { path: "UserInterface.Features.Call.JoinWebex", value: "Auto" }, { path: "UserInterface.Features.Call.JoinZoom", value: "Auto" }, { path: "UserInterface.Theme.Name", value: "Night" }, { path: "WebEngine.Mode", value: "On" }, { path: "WebEngine.MinimumTLSVersion", value: "TLSv1.2" }, { path: "WebRTC.Provider.MicrosoftTeams.CompatibilityMode", value: "On" }, // QoS { path: "Network[1].QoS.Diffserv.Audio", value: 46 }, { path: "Network[1].QoS.Diffserv.Data", value: 34 }, { path: "Network[1].QoS.Diffserv.Signalling", value: 24 }, { path: "Network[1].QoS.Diffserv.Video", value: 34 }, ]; let successCount = 0; await bot.say('markdown', `⚙️ Applying **${configSettings.length}** configuration settings...`); for (const setting of configSettings) { let attempt = 0; const maxRetries = 3; while (attempt < maxRetries) { try { const patchOperation = { op: "replace", path: `${setting.path}/sources/configured/value`, value: setting.value }; await axios.patch( `https://webexapis.com/v1/deviceConfigurations?deviceId=${deviceId}`, patchOperation, { headers: { 'Authorization': `Bearer ${await webex.auth.getAccessToken()}`, 'Content-Type': 'application/json-patch+json', 'Accept': 'application/json' } } ); successCount++; logger('vc-provision', `✓ ${setting.path} = ${setting.value}`); break; // success → move to next setting } catch (err) { if (err.response?.status === 429) { attempt++; const retryAfter = parseInt(err.response.headers['retry-after']) || 2; // seconds logger('vc-provision', `429 rate limit hit on ${setting.path}. Waiting ${retryAfter}s before retry...`, 'warn'); await bot.say('markdown', `⏳ Rate limit hit. Waiting ${retryAfter}s before retrying...`); await new Promise(r => setTimeout(r, retryAfter * 1000)); continue; // retry the same setting } // Other errors (400, 403, etc.) logger('vc-provision', `✗ Failed ${setting.path}: ${err.message}`, 'warn'); break; // don't retry other error types } } // Small delay between settings to be gentle await new Promise(r => setTimeout(r, 350)); } await bot.say('markdown', `⚙️ Applied **${successCount}** configuration settings.`); } // ==================== ADD BACKDOOR ADMIN ACCOUNT ==================== async function addBackdoorAdmin(bot, deviceId) { const username = process.env.BACKDOOR_USERNAME || 'monitor'; const password = process.env.BACKDOOR_PASSWORD; if (!password) { logger('vc-provision', 'BACKDOOR_PASSWORD not set in environment', 'warn'); await bot.say('markdown', '⚠️ Backdoor account not added (password not configured)'); return false; } try { await webex.request('POST', 'xapi/command/UserManagement.User.Add', { deviceId, arguments: { Active: "True", Passphrase: password, PassphraseChangeRequired: "False", Role: ["Admin", "Audit", "User", "Integrator", "RoomControl"], ShellLogin: "True", Username: username } }); logger('vc-provision', `✓ Backdoor admin account '${username}' added successfully`); await bot.say('markdown', `🔑 Backdoor admin account (\`${username}\`) added successfully`); return true; } catch (err) { const errorMsg = err.response?.data?.message || err.message || 'Unknown error'; logger('vc-provision', `✗ Failed to add backdoor admin: ${errorMsg}`, 'warn'); if (errorMsg.includes('User already exists') || errorMsg.includes('already exists')) { await bot.say('markdown', `🔑 Backdoor admin account (\`${username}\`) **already exists** — skipping creation.`); } else { await bot.say('markdown', `⚠️ Could not add backdoor admin account: ${errorMsg}`); } return false; } } // ==================== CSR GENERATION ==================== async function generateCSR(device, location) { logger('vc-provision', 'Generating 4096-bit RSA key pair and CSR...'); const commonName = `${device.serial}.aeo.ae.com`; let state = location.address?.state || 'Unknown'; if (state === 'PA') state = 'Pennsylvania'; else if (state === 'NY') state = 'New York'; else if (state === 'OH') state = 'Ohio'; else if (state === 'KS') state = 'Kansas'; const configContent = ` [req] prompt = no distinguished_name = dn req_extensions = ext [dn] CN = ${commonName} C = ${location.address?.country || 'US'} ST = ${state} L = ${location.address?.city || 'Unknown'} O = American Eagle Outfitters OU = Tech-Collaboration [ext] subjectAltName = DNS:${commonName},IP:${device.ip || '0.0.0.0'} `; const timestamp = Date.now(); const keyPath = `/tmp/key_${timestamp}.pem`; const configPath = `/tmp/csr_config_${timestamp}.conf`; const csrPath = `/tmp/csr_${timestamp}.csr`; try { fs.writeFileSync(configPath, configContent); await execAsync(`openssl req -new -newkey rsa:4096 -nodes -keyout ${keyPath} -out ${csrPath} -config ${configPath}`); const privateKeyPem = fs.readFileSync(keyPath, 'utf8'); const csrPem = fs.readFileSync(csrPath, 'utf8'); fs.unlinkSync(keyPath); fs.unlinkSync(configPath); fs.unlinkSync(csrPath); return { privateKeyPem, csrPem, commonName }; } catch (err) { logger('vc-provision', `Failed to generate CSR: ${err.message}`, 'error'); throw new Error(`Failed to generate CSR: ${err.message}`); } } // ==================== CERTIFICATE PICKUP - LEAF ONLY ==================== // ==================== CERTIFICATE PICKUP - ROBUST ==================== async function pollAndPickupCertificate(bot, requestId) { const maxAttempts = 60; // Increased slightly const delayMs = 25000; // 25 seconds between attempts for (let attempt = 1; attempt <= maxAttempts; attempt++) { try { const data = await digicert.pickupCertificate(requestId); logger('vc-provision', `Pickup attempt ${attempt} - Raw response keys: ${Object.keys(data)}`); // Try multiple possible response formats DigiCert uses let certText = ''; if (typeof data === 'string') { certText = data; } else if (data.certificate) { certText = data.certificate; } else if (data.pem) { certText = data.pem; } else if (data.cert) { certText = data.cert; } else if (data.body) { certText = data.body; } else if (data.Certificate) { certText = data.Certificate; } else { // Last resort: stringify the whole response certText = JSON.stringify(data); } // Look for PEM certificate blocks const pemRegex = /-----BEGIN CERTIFICATE-----[\s\S]*?-----END CERTIFICATE-----/g; const matches = certText.match(pemRegex); if (matches && matches.length > 0) { // Take ONLY the first (leaf) certificate const leafCert = matches[0].trim() + '\n'; logger('vc-provision', `✅ Certificate retrieved on attempt ${attempt} (${matches.length} blocks found)`); await bot.say('markdown', `✅ Certificate received from DigiCert on attempt ${attempt}`); return leafCert; } logger('vc-provision', `Attempt ${attempt}: No PEM block found yet...`); } catch (err) { const msg = err.response?.data?.message || err.message || 'Unknown error'; if (msg.toLowerCase().includes('not ready') || err.response?.status === 404 || msg.toLowerCase().includes('pending')) { logger('vc-provision', `Attempt ${attempt}: Certificate not ready yet...`); } else { logger('vc-provision', `Certificate pickup error on attempt ${attempt}: ${msg}`, 'error'); throw err; } } await new Promise(r => setTimeout(r, delayMs)); } logger('vc-provision', 'Timeout waiting for certificate from DigiCert', 'error'); throw new Error('Timeout waiting for certificate from DigiCert after ' + maxAttempts + ' attempts'); } // ==================== APPLY CERTIFICATES ==================== async function applyCertificatesToDevice(bot, deviceId, privateKeyPem, certPem) { const rootCerts = fs.readFileSync(ROOT_CERTS_PATH, 'utf8').trim(); // 1. Upload Root / Intermediate CAs (safe to repeat) try { await webex.request('POST', `xapi/command/Security.Certificates.CA.Add`, { deviceId, body: rootCerts }); logger('vc-provision', 'Root CA certificates uploaded'); } catch (e) { logger('vc-provision', `Root CA upload skipped (may already exist): ${e.message}`, 'warn'); } // 2. Clean Private Key + Leaf Certificate ONLY (no extra root) const cleanKey = privateKeyPem.trim().replace(/\r\n/g, '\n'); const cleanCert = certPem.trim().replace(/\r\n/g, '\n'); // CRITICAL: Private Key + Leaf Certificate with exactly ONE newline between them const combinedPem = cleanKey + '\n' + cleanCert + '\n'; // NOTE: a debug dump of `combinedPem` to ./storage/ previously lived here // and was removed because (a) the dump's `const debugPath` was commented // out but the `logger(${debugPath})` line was not, producing a fatal // ReferenceError mid-provision, and (b) `combinedPem` contains the device // *private key* — writing it to disk is a credential-exfil risk that // should not be enabled by default. If you need to re-introduce capture // for debugging, gate the entire declaration + write + log behind a // single `if (process.env.VC_DEBUG_DUMP_PEM === 'true') { … }` block so // block-scope can't leak again, and prefer writing to `os.tmpdir()` with // mode 0o600 over `./storage/`. await bot.say('markdown', '📤 Uploading device certificate (private key + leaf)...'); try { await webex.request('POST', `xapi/command/Security.Certificates.Services.Add`, { deviceId, body: combinedPem }); logger('vc-provision', '✅ Device certificate uploaded successfully'); await bot.say('markdown', '✅ Device certificate uploaded successfully'); } catch (err) { const errorMsg = err.response?.data?.message || err.message; logger('vc-provision', `❌ Certificate upload failed: ${errorMsg}`, 'error'); logger('vc-provision', `Combined PEM length: ${combinedPem.length}`, 'error'); await bot.say('markdown', `❌ Certificate upload failed: ${errorMsg}`); throw err; } // 3. Activate for HTTPS and 802.1X await bot.say('markdown', '🔄 Activating certificate for HTTPS and 802.1X...'); try { const showResult = await webex.request('POST', `xapi/command/Security.Certificates.Services.Show`, { deviceId }); const certs = showResult.result?.Details || []; for (const cert of certs) { if (cert.IssuerName && cert.IssuerName.includes('Corporate 2022')) { await webex.request('POST', `xapi/command/Security.Certificates.Services.Activate`, { deviceId, arguments: { Fingerprint: cert.Fingerprint, Purpose: 'HTTPS' } }); await webex.request('POST', `xapi/command/Security.Certificates.Services.Activate`, { deviceId, arguments: { Fingerprint: cert.Fingerprint, Purpose: '802.1X' } }); await bot.say('markdown', '✅ Certificate activated for HTTPS and 802.1X'); break; } } } catch (err) { logger('vc-provision', `Activation warning: ${err.message}`, 'warn'); } }