diff --git a/.env.example b/.env.example index c7048d6..737fbeb 100644 --- a/.env.example +++ b/.env.example @@ -45,6 +45,28 @@ JIRA_ASSETS_STORE_NUMBER_ATTRIBUTE_ID= # The custom field on the JSM request that holds the Store Assets reference. JIRA_STORE_CUSTOM_FIELD_ID=customfield_10261 +# --- Assets Stores cache (personal-PAT sync workaround) --- +# The service account is silently filtered out of Object Type 109 (see +# Forgejo issue #1). Until that's fixed, the app populates a local store-number +# -> objectId cache using a *personal* Atlassian PAT that has the right role. +# This PAT is used ONLY for reading the Stores schema; nothing that mutates +# Jira state uses it. +# +# ASSETS_SYNC_TOKEN should be exported at runtime from macOS Keychain via +# bin/load-assets-sync-secret.sh, NOT hardcoded here. It's listed only for +# completeness / documentation. +ASSETS_SYNC_EMAIL=you@ae.com +# ASSETS_SYNC_TOKEN= # loaded from Keychain by bin/load-assets-sync-secret.sh + +# Where the local cache lives on disk (JSON). Gitignored. Regenerable via +# POST /api/wxccai/admin/storesCache/refresh. +STORES_CACHE_PATH=./data/stores.json +# How often to run a full resync (hours). 0 disables the scheduler. +STORES_CACHE_REFRESH_HOURS=24 +# Cache is considered "stale" after this many hours; boot-time refresh fires +# if the on-disk snapshot is older than this. +STORES_CACHE_STALE_AFTER_HOURS=48 + # --- xAI (Grok) --- XAI_API_KEY=REPLACE_ME XAI_BASE_URL=https://api.x.ai/v1 diff --git a/.gitignore b/.gitignore index f7dff63..675f1b3 100644 --- a/.gitignore +++ b/.gitignore @@ -35,6 +35,11 @@ coverage/ tmp/ .tmp/ +# Stores cache and any other locally-generated app state. +# Never contains secrets, but is machine-specific and can be regenerated +# from Assets via POST /api/wxccai/admin/storesCache/refresh. +data/ + # ============================================= # Editor / IDE / OS # ============================================= diff --git a/README.md b/README.md index dfa49ac..e325e40 100644 --- a/README.md +++ b/README.md @@ -55,10 +55,48 @@ Base path: `/api/wxccai`. - `POST /issueTranscript/:jiraKey` — attaches audio + JSON transcript + human-readable transcript, then posts a restricted-visibility summary comment. +### Admin + +- `GET /admin/storesCache/status` — snapshot of the local Assets store cache: `{ storeCount, lastSyncAt, ageSeconds, syncing, assetsSyncConfigured, ... }`. Safe for health checks. +- `POST /admin/storesCache/refresh` — force an immediate resync via the personal PAT (see below). Takes a few seconds and returns the new status. + ### Debug (non-production only) - `GET /debug/assetsProbe?storeNumber=305` — runs several AQL variants against Jira Assets and returns visible schemas + object-type detail + a computed diagnosis. Returns 404 when `NODE_ENV=production`. +## Stores cache (Assets workaround) + +`createSSRequest` needs to translate a store number into an Assets object id. The shared service account is silently filtered out of the Store object type, so the app maintains a local `storeNumber → objectId` cache that's populated from a **personal Atlassian PAT** (a real human account with the right Assets role). The service account is still used for everything else (creating tickets, comments, attachments). + +**One-time setup:** + +```bash +# Store your PAT in macOS Keychain (never touches disk in cleartext) +security add-generic-password \ + -s jira-assets-sync \ + -a you@ae.com \ + -w '' \ + -U + +# Export the account name via .env / your shell +echo 'ASSETS_SYNC_EMAIL=you@ae.com' >> .env +``` + +**Run the server:** + +```bash +# Wrapper loads the PAT from Keychain into ASSETS_SYNC_TOKEN before exec +./bin/load-assets-sync-secret.sh npm start +``` + +On boot the app loads the on-disk cache at `data/stores.json`, kicks off a background refresh if the snapshot is missing or older than `STORES_CACHE_STALE_AFTER_HOURS`, and schedules a periodic full resync every `STORES_CACHE_REFRESH_HOURS`. `resolveStoreAssetReference` then serves lookups from memory (sub-ms) with a live PAT lookup as fallback for brand-new stores. + +Force a refresh at any time: + +```bash +curl -X POST http://localhost:1866/api/wxccai/admin/storesCache/refresh +``` + ## Jira Assets gotcha `createSSRequest` resolves a store number to an Assets object reference (`customfield_10261`). Two independent permission layers must both grant access, or every AQL query silently returns `total: 0`: diff --git a/bin/load-assets-sync-secret.sh b/bin/load-assets-sync-secret.sh new file mode 100755 index 0000000..224d43d --- /dev/null +++ b/bin/load-assets-sync-secret.sh @@ -0,0 +1,75 @@ +#!/usr/bin/env sh +# ----------------------------------------------------------------------------- +# load-assets-sync-secret.sh +# +# Loads the personal Atlassian PAT used for the Assets → Stores cache sync +# from macOS Keychain into ASSETS_SYNC_TOKEN, then execs whatever command you +# passed as arguments. +# +# Rationale: +# The service account is silently filtered out of Object Type 109 (Store +# Address / Hierarchy) — see Forgejo issue #1. The stores cache is populated +# using a personal PAT instead. This PAT authenticates as a real user and so +# should NEVER sit in .env in cleartext; the Keychain is a safer store. +# +# Setup (one-time, per machine): +# +# # 1. Store the token in Keychain +# security add-generic-password \ +# -s jira-assets-sync \ +# -a mcqueenj@ae.com \ +# -w '' \ +# -U +# +# # 2. Set the email (either exported here or in ~/.zshrc) +# export ASSETS_SYNC_EMAIL="mcqueenj@ae.com" +# +# Usage: +# +# ./bin/load-assets-sync-secret.sh npm start +# ./bin/load-assets-sync-secret.sh node src/app.js +# +# Environment variables (override defaults if needed): +# +# ASSETS_SYNC_KEYCHAIN_SERVICE Keychain service name (default: jira-assets-sync) +# ASSETS_SYNC_KEYCHAIN_ACCOUNT Keychain account name (default: value of $ASSETS_SYNC_EMAIL) +# ASSETS_SYNC_TOKEN If already set, skip the Keychain read entirely. +# ----------------------------------------------------------------------------- + +set -eu + +if [ -z "${ASSETS_SYNC_TOKEN:-}" ]; then + SERVICE="${ASSETS_SYNC_KEYCHAIN_SERVICE:-jira-assets-sync}" + ACCOUNT="${ASSETS_SYNC_KEYCHAIN_ACCOUNT:-${ASSETS_SYNC_EMAIL:-}}" + + if [ -z "$ACCOUNT" ]; then + printf 'load-assets-sync-secret.sh: neither ASSETS_SYNC_TOKEN nor an account name is set.\n' >&2 + printf ' Export ASSETS_SYNC_EMAIL=you@ae.com or ASSETS_SYNC_KEYCHAIN_ACCOUNT=.\n' >&2 + exit 1 + fi + + if ! command -v security >/dev/null 2>&1; then + printf 'load-assets-sync-secret.sh: `security` not found (this script is macOS-only).\n' >&2 + printf ' On Linux/prod, export ASSETS_SYNC_TOKEN directly from your secret manager.\n' >&2 + exit 1 + fi + + if ! ASSETS_SYNC_TOKEN=$(security find-generic-password -s "$SERVICE" -a "$ACCOUNT" -w 2>/dev/null); then + printf 'load-assets-sync-secret.sh: Keychain lookup failed for service="%s" account="%s".\n' "$SERVICE" "$ACCOUNT" >&2 + printf ' Store the token with:\n' >&2 + printf ' security add-generic-password -s %s -a %s -w '\'''\'' -U\n' "$SERVICE" "$ACCOUNT" >&2 + exit 1 + fi + export ASSETS_SYNC_TOKEN +fi + +# Only echoes existence, never the token itself. +printf 'load-assets-sync-secret.sh: ASSETS_SYNC_TOKEN loaded (%d chars) for %s\n' \ + "${#ASSETS_SYNC_TOKEN}" "${ASSETS_SYNC_EMAIL:-}" >&2 + +if [ $# -eq 0 ]; then + printf 'load-assets-sync-secret.sh: no command given; exiting after loading token.\n' >&2 + exit 0 +fi + +exec "$@" diff --git a/src/app.js b/src/app.js index 96e442c..21a9ecd 100644 --- a/src/app.js +++ b/src/app.js @@ -2,6 +2,7 @@ import express from 'express'; import config from './config/index.js'; import wxccRoutes from './routes/wxccRoutes.js'; import { getDetailedHealth } from './services/healthService.js'; +import * as storesCache from './services/jira/storesCache.js'; import logger from './utilities/logger.js'; const app = express(); @@ -106,6 +107,8 @@ app.get('/', (req, res) => { closeTicket: "POST /api/wxccai/ticket/:key/close body: { transitionName?, resolution?, comment?, internal? }", createSSRequest: "POST /api/wxccai/createSSRequest (store support ticket creator; storeNumber auto-resolved via Assets)", ssRequestTypes: "GET /api/wxccai/ssRequestTypes (lists exact subType values + config notes)", + storesCacheStatus: "GET /api/wxccai/admin/storesCache/status (Assets store cache state)", + storesCacheRefresh: "POST /api/wxccai/admin/storesCache/refresh (force a resync via personal PAT)", assetsProbe: "GET /api/wxccai/debug/assetsProbe?storeNumber=305 (non-production only)" } }); @@ -131,4 +134,10 @@ app.use((req, res) => { app.listen(config.port, () => { logger.info(`Server running on port ${config.port} in ${config.nodeEnv} mode`); + + // Kick off the stores cache: load from disk, schedule periodic refresh, + // and (if stale) start a background full refresh. Never blocks startup. + storesCache.init().catch(err => { + logger.error('storesCache.init failed at boot', { error: err.message }); + }); }); \ No newline at end of file diff --git a/src/config/index.js b/src/config/index.js index df0d20f..509ae60 100644 --- a/src/config/index.js +++ b/src/config/index.js @@ -1,9 +1,23 @@ import dotenv from 'dotenv'; +import path from 'node:path'; dotenv.config(); const cloudId = process.env.JIRA_CLOUD_ID?.trim(); let baseUrl = process.env.JIRA_BASE_URL?.trim(); +const intFromEnv = (name, fallback) => { + const raw = process.env[name]; + if (raw === undefined || raw === null || raw === '') return fallback; + const n = Number.parseInt(raw, 10); + return Number.isFinite(n) ? n : fallback; +}; + +const boolFromEnv = (name, fallback) => { + const raw = process.env[name]; + if (raw === undefined || raw === null || raw === '') return fallback; + return /^(1|true|yes|on)$/i.test(String(raw).trim()); +}; + if (cloudId) { // When JIRA_CLOUD_ID is provided, use the api.atlassian.com/ex/jira gateway form. // This is the current style for addressing a specific Jira Cloud site (e.g. for @@ -56,8 +70,36 @@ export const config = { // The custom field ID on the request that holds the Store Assets reference storeCustomFieldId: process.env.JIRA_STORE_CUSTOM_FIELD_ID || 'customfield_10261', + + // --- Personal-account credentials for the Stores cache sync --- + // Compartmentalized from the service account above. Used ONLY by + // src/services/jira/assetsSyncClient.js to populate storesCache. This + // exists because the service account is silently filtered out for the + // Store object type (see Forgejo issue #1 for the underlying + // permission problem). Never used for anything that mutates state on + // Atlassian's side. + // + // The token is expected to be exported into the process env from macOS + // Keychain via bin/load-assets-sync-secret.sh — it should NOT sit in + // .env in cleartext. + assetsSyncEmail: process.env.ASSETS_SYNC_EMAIL?.trim() || null, + assetsSyncToken: process.env.ASSETS_SYNC_TOKEN || null, }, - + + // Local cache of the Assets Store schema. Backed by a JSON file on disk; + // refreshed periodically via the personal PAT above. + storesCache: { + enabled: boolFromEnv('STORES_CACHE_ENABLED', true), + // Where the on-disk snapshot lives. Default is data/stores.json under + // the process cwd; gitignored. + path: process.env.STORES_CACHE_PATH?.trim() + || path.resolve(process.cwd(), 'data', 'stores.json'), + refreshIntervalHours: intFromEnv('STORES_CACHE_REFRESH_HOURS', 24), + staleAfterHours: intFromEnv('STORES_CACHE_STALE_AFTER_HOURS', 48), + pageSize: intFromEnv('STORES_CACHE_PAGE_SIZE', 500), + maxPages: intFromEnv('STORES_CACHE_MAX_PAGES', 200), + }, + xai: { apiKey: process.env.XAI_API_KEY, baseUrl: process.env.XAI_BASE_URL || 'https://api.x.ai/v1', diff --git a/src/routes/wxccRoutes.js b/src/routes/wxccRoutes.js index d03e2b3..a7aa0e4 100644 --- a/src/routes/wxccRoutes.js +++ b/src/routes/wxccRoutes.js @@ -312,6 +312,40 @@ router.post('/wxccai/ticket/:key/close', async (req, res) => { } }); +// ======================== +// Stores cache admin +// GET /api/wxccai/admin/storesCache/status — snapshot of cache state +// POST /api/wxccai/admin/storesCache/refresh — force an immediate resync +// +// These are admin endpoints (not user-facing). The status endpoint is safe to +// hit from a health check; the refresh endpoint triggers an AQL walk of the +// entire Store object type via the personal PAT and takes a few seconds on a +// warm connection. Refresh is a no-op if credentials aren't configured or a +// sync is already in progress. +// ======================== +router.get('/wxccai/admin/storesCache/status', (req, res) => { + try { + res.json(jiraService.storesCache.status()); + } catch (err) { + logger.error('storesCache status failed', { error: err.message }); + res.status(500).json({ error: err.message }); + } +}); + +router.post('/wxccai/admin/storesCache/refresh', async (req, res) => { + try { + const result = await jiraService.storesCache.refresh({ force: true }); + res.json({ success: true, ...result, status: jiraService.storesCache.status() }); + } catch (err) { + logger.error('storesCache refresh failed', { error: err.message }); + res.status(500).json({ + success: false, + error: err.message, + status: jiraService.storesCache.status(), + }); + } +}); + // ======================== // Assets diagnostic (non-production only) // GET /api/wxccai/debug/assetsProbe?storeNumber=305 diff --git a/src/services/jira/assets.js b/src/services/jira/assets.js index 1217874..776ac54 100644 --- a/src/services/jira/assets.js +++ b/src/services/jira/assets.js @@ -15,6 +15,8 @@ import axios from 'axios'; import config from '../../config/index.js'; import logger from '../../utilities/logger.js'; import { jiraClient } from './client.js'; +import * as storesCache from './storesCache.js'; +import { assetsAql as syncAssetsAql, isAssetsSyncConfigured } from './assetsSyncClient.js'; /** * Resolve the Assets workspace id. Prefers the explicit env var; otherwise @@ -236,9 +238,20 @@ function summarizeAssetsObject(obj) { /** * Resolve a store number (e.g. "00305" or 305) to the Assets object reference - * for the Store custom field in a JSM request: + * used for the Store custom field on a JSM request: * customfield_10261: [ { "objectId": "82288" } ] - * Tries multiple AQL variants (attribute name; attribute id if configured). + * + * Resolution order (each step logged so we can tell which path served the + * lookup): + * 1. Local stores cache (populated by the personal-PAT sync). Sub-ms; no + * external call. + * 2. Live AQL via the personal PAT — handles brand-new stores that landed + * between scheduled syncs. + * 3. Legacy AQL via the service account (kept as a fallback; currently + * blocked by the Object-Type-109 permission wall tracked in Forgejo #1, + * but will start working when that's resolved). + * + * Only throws once all three paths have missed. */ export async function resolveStoreAssetReference(rawStoreNumber) { if (!rawStoreNumber) return null; @@ -248,6 +261,50 @@ export async function resolveStoreAssetReference(rawStoreNumber) { const attribute = config.jira.assetsStoreNumberAttribute || 'Store Number'; const attrId = config.jira.assetsStoreNumberAttributeId; + // 1. Cache hit — the common case after the first sync. + const cached = storesCache.get(normalized); + if (cached) { + logger.info('Resolved store via cache', { + storeNumber: normalized, + objectId: cached.objectId, + cacheAge: storesCache.status().ageSeconds, + }); + return [{ objectId: String(cached.objectId) }]; + } + + // 2. Live lookup via the personal-PAT sync client (handles brand-new + // stores not yet in the cache). + if (isAssetsSyncConfigured()) { + const ql = `objectTypeId = ${objectTypeId} AND "${attribute}" = "${normalized}"`; + logger.info('Cache miss; trying live PAT lookup for store', { storeNumber: normalized, ql }); + const r = await syncAssetsAql(ql, { resultPerPage: 1, includeAttributes: false }); + if (r.status === 200) { + const values = Array.isArray(r.data?.values) ? r.data.values : []; + if (values.length > 0) { + const objectId = extractObjectIdFromResponse(r.data); + if (objectId) { + logger.info('Resolved store via live PAT lookup', { + storeNumber: normalized, + objectId: String(objectId), + }); + return [{ objectId: String(objectId) }]; + } + } + logger.warn('Live PAT lookup returned 200 but no matching object', { + storeNumber: normalized, + total: r.data?.total, + }); + } else { + logger.warn('Live PAT lookup failed', { + storeNumber: normalized, + status: r.status, + error: r.error, + }); + } + } + + // 3. Fallback: legacy service-account AQL. Kept behind the cache/PAT + // layers so it costs nothing on the happy path. const queries = [ `objectTypeId = ${objectTypeId} AND "${attribute}" = "${normalized}"` ]; @@ -258,7 +315,7 @@ export async function resolveStoreAssetReference(rawStoreNumber) { let lastResult = null; for (const qlQuery of queries) { - logger.info('Assets AQL lookup for store number', { qlQuery, storeNumber: normalized }); + logger.info('Assets AQL lookup for store number (service-account fallback)', { qlQuery, storeNumber: normalized }); const result = await runAssetsAql(qlQuery, { resultPerPage: 1, includeAttributes: true }); lastResult = result; @@ -292,14 +349,21 @@ export async function resolveStoreAssetReference(rawStoreNumber) { continue; } - logger.info('Resolved store to Assets object', { storeNumber: normalized, objectId: String(objectId) }); + logger.info('Resolved store via service-account AQL fallback', { storeNumber: normalized, objectId: String(objectId) }); return [{ objectId: String(objectId) }]; } + // All three paths missed — surface a message that tells the caller which + // routes were tried and what the cache looked like. + const cacheStatus = storesCache.status(); const total = lastResult?.data?.total ?? 'unknown'; + const cacheHint = cacheStatus.assetsSyncConfigured + ? `cache has ${cacheStatus.storeCount} entries, last synced ${cacheStatus.lastSyncAt || 'never'}` + : 'cache is not populated (ASSETS_SYNC_EMAIL/ASSETS_SYNC_TOKEN not configured)'; throw new Error( `Failed to resolve Store Number ${normalized} via Assets (objectTypeId=${objectTypeId}). ` + - `No results or unparseable id across all variants. Last total=${total}` + `Tried: local cache, live PAT lookup, service-account AQL. ${cacheHint}. Last service-account total=${total}. ` + + `Try POST /api/wxccai/admin/storesCache/refresh to force a resync.` ); } diff --git a/src/services/jira/assetsSyncClient.js b/src/services/jira/assetsSyncClient.js new file mode 100644 index 0000000..6e990c7 --- /dev/null +++ b/src/services/jira/assetsSyncClient.js @@ -0,0 +1,136 @@ +// Dedicated Jira Assets client that authenticates as a *personal* Atlassian +// account (email + PAT), not the shared service account used everywhere else. +// +// Why this exists: +// The service account has schema-level read on the Stores schema but is +// silently filtered out for objects in Object Type 109 (Store Address / +// Hierarchy). Every direct API path (raw Assets, site gateway, servicedeskapi +// CMDB proxy) is either 403 or returns total=0. A personal account with the +// right Assets role sees the objects fine, so this client is used *only* by +// the store cache sync — never for anything that mutates state on Atlassian's +// side. +// +// Credentials come from config.jira.assetsSyncEmail / assetsSyncToken. Those +// are populated from env vars (which are typically exported from macOS +// Keychain via bin/load-assets-sync-secret.sh). If either is missing this +// module is inert and the caller falls back to the service account path. +import axios from 'axios'; +import config from '../../config/index.js'; +import logger from '../../utilities/logger.js'; + +const ASSETS_HOST = 'https://api.atlassian.com'; + +let cachedAuthHeader = null; +let warnedMissing = false; + +function buildAuthHeader() { + const email = config.jira.assetsSyncEmail; + const token = config.jira.assetsSyncToken; + if (!email || !token) return null; + const encoded = Buffer.from(`${email}:${token}`).toString('base64'); + return `Basic ${encoded}`; +} + +function getAuthHeader() { + if (cachedAuthHeader) return cachedAuthHeader; + cachedAuthHeader = buildAuthHeader(); + if (!cachedAuthHeader && !warnedMissing) { + logger.warn('Assets sync credentials not configured (ASSETS_SYNC_EMAIL / ASSETS_SYNC_TOKEN). Stores cache sync is disabled; resolveStoreAssetReference will fall back to the service-account AQL path.'); + warnedMissing = true; + } + return cachedAuthHeader; +} + +/** + * True iff both an email and a PAT are configured for the personal-account + * Assets sync path. + */ +export function isAssetsSyncConfigured() { + return !!getAuthHeader(); +} + +/** + * Best-effort masked identity for logging / status endpoints. Never returns + * the token. + */ +export function describeSyncIdentity() { + const email = config.jira.assetsSyncEmail || null; + return { + configured: isAssetsSyncConfigured(), + email: email + ? (email.includes('@') + ? `${email.slice(0, 3)}…@${email.split('@')[1]}` + : `${email.slice(0, 3)}…`) + : null, + }; +} + +/** + * Low-level AQL POST via the personal PAT. Never throws on non-2xx — always + * returns `{ status, statusText, data, headers, requestUrl, requestBody, error }`. + * + * `qlQuery` raw AQL string, e.g. `objectTypeId = 109` + * `opts` { page, resultPerPage, includeAttributes, extraBody, timeoutMs } + */ +export async function assetsAql(qlQuery, opts = {}) { + const auth = getAuthHeader(); + if (!auth) { + return { + status: 'not_configured', + statusText: 'ASSETS_SYNC_EMAIL / ASSETS_SYNC_TOKEN not set', + data: null, + error: 'assets sync credentials not configured', + }; + } + + const workspaceId = config.jira.assetsWorkspaceId; + if (!workspaceId) { + return { + status: 'not_configured', + statusText: 'JIRA_ASSETS_WORKSPACE_ID not set', + data: null, + error: 'assets workspace id not configured', + }; + } + + const { + page = 1, + resultPerPage = 500, + includeAttributes = true, + extraBody = {}, + timeoutMs = 30000, + } = opts; + + const url = `${ASSETS_HOST}/jsm/assets/workspace/${workspaceId}/v1/object/aql?page=${page}&resultPerPage=${resultPerPage}&includeAttributes=${includeAttributes}`; + const body = { qlQuery, ...extraBody }; + + try { + const resp = await axios.post(url, body, { + headers: { + Authorization: auth, + 'Content-Type': 'application/json', + Accept: 'application/json', + }, + timeout: timeoutMs, + validateStatus: () => true, + }); + return { + status: resp.status, + statusText: resp.statusText, + data: resp.data ?? null, + headers: resp.headers, + requestUrl: url, + requestBody: body, + }; + } catch (err) { + return { + status: err.response?.status || 'network', + statusText: err.response?.statusText || err.code || 'error', + data: err.response?.data || null, + headers: err.response?.headers || {}, + requestUrl: url, + requestBody: body, + error: err.message, + }; + } +} diff --git a/src/services/jira/storesCache.js b/src/services/jira/storesCache.js new file mode 100644 index 0000000..928580c --- /dev/null +++ b/src/services/jira/storesCache.js @@ -0,0 +1,320 @@ +// In-memory + on-disk cache of `storeNumber → Assets objectId`. +// +// Populated via assetsSyncClient (personal PAT). This is the primary lookup +// path used by resolveStoreAssetReference — the raw service-account AQL call +// is only a fallback for the (currently-blocked) scenario where the cache is +// unavailable and the service account has been granted enough permission to +// query the store type directly. +// +// Storage: +// In-memory: Map +// On-disk: data/stores.json (see config.storesCache.path). Gitignored. +// +// Lifecycle: +// init() — called at boot; loads disk cache and kicks off +// background refresh if missing/stale, then schedules +// periodic refresh via setInterval. +// refresh({force}) — full paginated resync via personal PAT. +// get(storeNumber) — sync lookup, returns entry or null. +// status() — for the admin endpoint. +// +// Store objects live in ObjectType 109 in schema 68. The `label` on those +// objects is the padded store number (verified against real objects in the +// tenant); the `Store Number` attribute is a secondary source we fall back to. +import fs from 'node:fs/promises'; +import path from 'node:path'; +import config from '../../config/index.js'; +import logger from '../../utilities/logger.js'; +import { assetsAql, isAssetsSyncConfigured, describeSyncIdentity } from './assetsSyncClient.js'; + +const state = { + cache: new Map(), + lastSyncAt: null, + lastError: null, + syncing: false, + scheduleTimer: null, + loadedFromDisk: false, +}; + +function normalizeStoreNumber(raw) { + if (raw === null || raw === undefined) return null; + const s = String(raw).trim(); + if (!s) return null; + if (!/^\d+$/.test(s)) return null; + return s.padStart(5, '0'); +} + +function extractStoreNumberFromObject(obj, storeNumberAttrName) { + // Primary: the label is the padded store number on type 109 objects. + const label = obj?.label || obj?.name; + if (label && /^\d+$/.test(String(label).trim())) { + return String(label).trim().padStart(5, '0'); + } + // Secondary: check the named attribute. + const attrs = Array.isArray(obj?.attributes) ? obj.attributes : []; + for (const attr of attrs) { + const meta = attr?.objectTypeAttribute || attr?.typeAttribute || {}; + if (meta?.name === storeNumberAttrName) { + const vals = Array.isArray(attr?.objectAttributeValues) ? attr.objectAttributeValues : []; + for (const v of vals) { + const cand = v?.displayValue ?? v?.value ?? v?.searchValue; + if (cand && /^\d+$/.test(String(cand).trim())) { + return String(cand).trim().padStart(5, '0'); + } + } + } + } + return null; +} + +/** + * Sync (in-memory) lookup. Returns null on miss; never throws. + */ +export function get(storeNumber) { + const n = normalizeStoreNumber(storeNumber); + if (!n) return null; + return state.cache.get(n) || null; +} + +/** + * Snapshot for the admin status endpoint. Safe to serialize to JSON. + */ +export function status() { + const lastSyncEpoch = state.lastSyncAt ? new Date(state.lastSyncAt).getTime() : null; + return { + storeCount: state.cache.size, + lastSyncAt: state.lastSyncAt, + ageSeconds: lastSyncEpoch ? Math.floor((Date.now() - lastSyncEpoch) / 1000) : null, + syncing: state.syncing, + lastError: state.lastError, + loadedFromDisk: state.loadedFromDisk, + assetsSyncConfigured: isAssetsSyncConfigured(), + syncIdentity: describeSyncIdentity(), + cachePath: config.storesCache.path, + refreshIntervalHours: config.storesCache.refreshIntervalHours, + staleAfterHours: config.storesCache.staleAfterHours, + }; +} + +async function loadFromDisk() { + const cachePath = config.storesCache.path; + try { + const raw = await fs.readFile(cachePath, 'utf8'); + const parsed = JSON.parse(raw); + if (parsed && typeof parsed === 'object' && parsed.entries) { + state.cache = new Map(Object.entries(parsed.entries)); + state.lastSyncAt = parsed.lastSyncAt || null; + state.loadedFromDisk = true; + logger.info('Loaded stores cache from disk', { + path: cachePath, + storeCount: state.cache.size, + lastSyncAt: state.lastSyncAt, + }); + return true; + } + logger.warn('Stores cache file present but shape unexpected; ignoring', { path: cachePath }); + } catch (e) { + if (e.code === 'ENOENT') { + logger.info('No stores cache on disk yet; will populate on first sync', { path: cachePath }); + } else { + logger.warn('Failed to load stores cache from disk', { path: cachePath, error: e.message }); + } + } + return false; +} + +async function saveToDisk() { + const cachePath = config.storesCache.path; + const dir = path.dirname(cachePath); + try { + await fs.mkdir(dir, { recursive: true }); + const payload = { + version: 1, + lastSyncAt: state.lastSyncAt, + storeCount: state.cache.size, + entries: Object.fromEntries(state.cache), + }; + // Atomic-ish write: to tmp then rename, avoids readers seeing a half-written file. + const tmp = `${cachePath}.tmp`; + await fs.writeFile(tmp, JSON.stringify(payload, null, 2), 'utf8'); + await fs.rename(tmp, cachePath); + logger.debug('Persisted stores cache to disk', { path: cachePath, storeCount: state.cache.size }); + } catch (e) { + logger.warn('Failed to persist stores cache to disk', { path: cachePath, error: e.message }); + } +} + +/** + * Full paginated resync of all objects in the configured store object type + * via the personal PAT. Rebuilds the in-memory cache atomically (only swaps + * once every page has been read successfully) and then writes to disk. + * + * Never called reentrantly — if a refresh is already in flight, returns + * { skipped: true, reason: 'already_syncing' }. + */ +export async function refresh({ force = false } = {}) { + if (state.syncing) { + logger.info('Stores cache refresh already in progress; ignoring duplicate call'); + return { skipped: true, reason: 'already_syncing' }; + } + if (!isAssetsSyncConfigured()) { + return { skipped: true, reason: 'not_configured' }; + } + + state.syncing = true; + state.lastError = null; + const started = Date.now(); + const objectTypeId = config.jira.assetsStoreObjectTypeId || '109'; + const storeNumberAttr = config.jira.assetsStoreNumberAttribute || 'Store Number'; + const pageSize = config.storesCache.pageSize || 500; + const maxPages = config.storesCache.maxPages || 200; // safety cap; 200*500 = 100k stores + + try { + const newCache = new Map(); + let page = 1; + let seenTotal = 0; + let expectedTotal = null; + let orphaned = 0; + + while (page <= maxPages) { + const r = await assetsAql(`objectTypeId = ${objectTypeId}`, { + page, + resultPerPage: pageSize, + includeAttributes: true, + }); + + if (r.status !== 200) { + const bodySnippet = typeof r.data === 'object' + ? JSON.stringify(r.data).slice(0, 400) + : String(r.data).slice(0, 400); + throw new Error( + `Assets AQL page ${page} returned HTTP ${r.status} (${r.statusText}). Body: ${bodySnippet}` + ); + } + + const values = Array.isArray(r.data?.values) ? r.data.values : []; + if (expectedTotal === null && typeof r.data?.total === 'number') { + expectedTotal = r.data.total; + } + + for (const obj of values) { + const storeNumber = extractStoreNumberFromObject(obj, storeNumberAttr); + if (!storeNumber || !obj?.id) { + orphaned++; + continue; + } + newCache.set(storeNumber, { + objectId: String(obj.id), + objectKey: obj.objectKey || null, + label: obj.label || obj.name || null, + syncedAt: new Date().toISOString(), + }); + } + seenTotal += values.length; + + const isLast = r.data?.isLast === true + || values.length < pageSize + || (expectedTotal !== null && seenTotal >= expectedTotal); + + logger.debug('Stores cache sync page', { + page, + rowsThisPage: values.length, + seenTotal, + expectedTotal, + mappedSoFar: newCache.size, + orphanedSoFar: orphaned, + isLast, + }); + + if (values.length === 0 || isLast) break; + page++; + } + + if (page > maxPages) { + logger.warn('Stores cache sync hit page cap; some stores may be missing', { + maxPages, + pageSize, + seenTotal, + }); + } + + // Only swap once we've read every page successfully. + state.cache = newCache; + state.lastSyncAt = new Date().toISOString(); + await saveToDisk(); + + const durationMs = Date.now() - started; + logger.info('Stores cache refresh complete', { + storeCount: state.cache.size, + orphaned, + pagesRead: page, + durationMs, + forced: force, + }); + + return { + ok: true, + storeCount: state.cache.size, + orphaned, + pagesRead: page, + durationMs, + }; + } catch (e) { + state.lastError = { message: e.message, at: new Date().toISOString() }; + logger.error('Stores cache refresh failed', { error: e.message }); + throw e; + } finally { + state.syncing = false; + } +} + +/** + * Load the disk cache, kick off a background refresh if missing or stale, and + * schedule periodic refreshes. Idempotent — safe to call more than once. + * + * Never throws. Individual failures are logged and surfaced through status(). + */ +export async function init() { + if (!config.storesCache.enabled) { + logger.info('Stores cache disabled via config; skipping init'); + return; + } + + await loadFromDisk(); + + const staleAfterMs = (config.storesCache.staleAfterHours || 48) * 3600 * 1000; + const shouldRefreshNow = !state.lastSyncAt + || (Date.now() - new Date(state.lastSyncAt).getTime()) > staleAfterMs; + + if (shouldRefreshNow && isAssetsSyncConfigured()) { + logger.info('Stores cache is missing or stale; starting background refresh'); + refresh().catch(e => logger.error('Initial stores cache refresh failed', { error: e.message })); + } else if (shouldRefreshNow) { + logger.warn('Stores cache is missing or stale but assets sync is not configured; skipping initial refresh'); + } + + const intervalHours = config.storesCache.refreshIntervalHours; + if (intervalHours > 0) { + if (state.scheduleTimer) clearInterval(state.scheduleTimer); + state.scheduleTimer = setInterval(() => { + if (!isAssetsSyncConfigured()) return; + refresh().catch(e => logger.error('Scheduled stores cache refresh failed', { error: e.message })); + }, intervalHours * 3600 * 1000); + state.scheduleTimer.unref?.(); // don't block process exit for tests + logger.info('Stores cache: periodic refresh scheduled', { intervalHours }); + } +} + +/** + * Test / shutdown hook. Clears the schedule timer and the in-memory map. + * Does NOT delete the on-disk cache. + */ +export function shutdown() { + if (state.scheduleTimer) clearInterval(state.scheduleTimer); + state.scheduleTimer = null; + state.cache = new Map(); + state.lastSyncAt = null; + state.lastError = null; + state.syncing = false; + state.loadedFromDisk = false; +} diff --git a/src/services/jiraService.js b/src/services/jiraService.js index 1ffa0fc..f03fd01 100644 --- a/src/services/jiraService.js +++ b/src/services/jiraService.js @@ -45,6 +45,11 @@ export { probeAssetsForStore, } from './jira/assets.js'; +// Stores cache (Assets store number -> objectId), fed by a personal-PAT sync +// to work around the service-account permission block on Object Type 109. +export * as storesCache from './jira/storesCache.js'; +export { isAssetsSyncConfigured, describeSyncIdentity } from './jira/assetsSyncClient.js'; + export { REQUEST_TYPE_MAP, getSupportedSSSubTypes,