From 8ac64ba0777184d5fa07339e22a94c7dc8c315ef Mon Sep 17 00:00:00 2001 From: Kelly Date: Sun, 7 Dec 2025 11:04:12 -0700 Subject: [PATCH] feat(cannaiq): Add Workers Dashboard and visibility tracking MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Workers Dashboard: - New /workers route with two-pane layout - Workers table showing Alice, Henry, Bella, Oscar with role badges - Run history with visibility stats (lost/restored counts) - "Run Now" action to trigger workers immediately Migrations: - 057: Add visibility tracking columns (visibility_lost, visibility_lost_at, visibility_restored_at) - 058: Add ID resolution columns for Henry worker - 059: Add job queue columns (max_retries, retry_count, worker_id, locked_at, locked_by) Backend fixes: - Add httpStatus to CrawlResult interface for error classification - Fix pool.ts typing for event listener - Update completeJob to accept visibility stats in metadata Frontend fixes: - Fix NationalDashboard crash with safe formatMoney helper - Fix OrchestratorDashboard/Stores StoreInfo type mismatches - Add workerName/workerRole to getDutchieAZSchedules API type 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 --- .../057_visibility_tracking_columns.sql | 64 + .../058_add_id_resolution_columns.sql | 46 + backend/migrations/059_job_queue_columns.sql | 67 + backend/src/db/pool.ts | 94 ++ backend/src/dutchie-az/services/job-queue.ts | 172 ++- .../dutchie-az/services/product-crawler.ts | 198 ++- backend/src/dutchie-az/services/worker.ts | 369 +++++- .../__tests__/state-query-service.test.ts | 339 +++++ backend/src/multi-state/index.ts | 15 + backend/src/multi-state/routes.ts | 451 +++++++ .../src/multi-state/state-query-service.ts | 643 ++++++++++ backend/src/multi-state/types.ts | 199 +++ cannaiq/src/App.tsx | 38 + cannaiq/src/components/Layout.tsx | 93 +- .../src/components/StoreOrchestratorPanel.tsx | 1115 +++++++++++++++++ cannaiq/src/components/WorkerRoleBadge.tsx | 138 ++ cannaiq/src/lib/api.ts | 1018 +++++++++++++++ cannaiq/src/pages/NationalDashboard.tsx | 378 ++++++ cannaiq/src/pages/OrchestratorDashboard.tsx | 472 +++++++ cannaiq/src/pages/OrchestratorStores.tsx | 264 ++++ .../src/pages/ScraperOverviewDashboard.tsx | 485 +++++++ cannaiq/src/pages/WorkersDashboard.tsx | 498 ++++++++ 22 files changed, 7022 insertions(+), 134 deletions(-) create mode 100644 backend/migrations/057_visibility_tracking_columns.sql create mode 100644 backend/migrations/058_add_id_resolution_columns.sql create mode 100644 backend/migrations/059_job_queue_columns.sql create mode 100644 backend/src/db/pool.ts create mode 100644 backend/src/multi-state/__tests__/state-query-service.test.ts create mode 100644 backend/src/multi-state/index.ts create mode 100644 backend/src/multi-state/routes.ts create mode 100644 backend/src/multi-state/state-query-service.ts create mode 100644 backend/src/multi-state/types.ts create mode 100644 cannaiq/src/components/StoreOrchestratorPanel.tsx create mode 100644 cannaiq/src/components/WorkerRoleBadge.tsx create mode 100644 cannaiq/src/pages/NationalDashboard.tsx create mode 100644 cannaiq/src/pages/OrchestratorDashboard.tsx create mode 100644 cannaiq/src/pages/OrchestratorStores.tsx create mode 100644 cannaiq/src/pages/ScraperOverviewDashboard.tsx create mode 100644 cannaiq/src/pages/WorkersDashboard.tsx diff --git a/backend/migrations/057_visibility_tracking_columns.sql b/backend/migrations/057_visibility_tracking_columns.sql new file mode 100644 index 00000000..b13e53c3 --- /dev/null +++ b/backend/migrations/057_visibility_tracking_columns.sql @@ -0,0 +1,64 @@ +-- Migration 057: Add visibility tracking columns to dutchie_products +-- +-- Supports Bella (Product Sync) worker visibility-loss tracking: +-- - visibility_lost: TRUE when product disappears from GraphQL feed +-- - visibility_lost_at: Timestamp when product first went missing +-- - visibility_restored_at: Timestamp when product reappeared +-- +-- These columns enable tracking of products that temporarily or permanently +-- disappear from Dutchie GraphQL API responses. + +-- ============================================================ +-- 1. ADD VISIBILITY TRACKING COLUMNS TO dutchie_products +-- ============================================================ + +ALTER TABLE dutchie_products + ADD COLUMN IF NOT EXISTS visibility_lost BOOLEAN DEFAULT FALSE, + ADD COLUMN IF NOT EXISTS visibility_lost_at TIMESTAMPTZ, + ADD COLUMN IF NOT EXISTS visibility_restored_at TIMESTAMPTZ; + +COMMENT ON COLUMN dutchie_products.visibility_lost IS 'TRUE when product is missing from GraphQL feed'; +COMMENT ON COLUMN dutchie_products.visibility_lost_at IS 'Timestamp when product first went missing from feed'; +COMMENT ON COLUMN dutchie_products.visibility_restored_at IS 'Timestamp when product reappeared after being missing'; + +-- ============================================================ +-- 2. CREATE INDEX FOR VISIBILITY QUERIES +-- ============================================================ + +CREATE INDEX IF NOT EXISTS idx_dutchie_products_visibility_lost + ON dutchie_products(visibility_lost) + WHERE visibility_lost = TRUE; + +CREATE INDEX IF NOT EXISTS idx_dutchie_products_visibility_lost_at + ON dutchie_products(visibility_lost_at) + WHERE visibility_lost_at IS NOT NULL; + +-- ============================================================ +-- 3. CREATE VIEW FOR VISIBILITY ANALYTICS +-- ============================================================ + +CREATE OR REPLACE VIEW v_visibility_summary AS +SELECT + d.id AS dispensary_id, + d.name AS dispensary_name, + d.state, + COUNT(dp.id) AS total_products, + COUNT(dp.id) FILTER (WHERE dp.visibility_lost = TRUE) AS visibility_lost_count, + COUNT(dp.id) FILTER (WHERE dp.visibility_lost = FALSE OR dp.visibility_lost IS NULL) AS visible_count, + COUNT(dp.id) FILTER (WHERE dp.visibility_restored_at IS NOT NULL) AS restored_count, + MAX(dp.visibility_lost_at) AS latest_loss_at, + MAX(dp.visibility_restored_at) AS latest_restore_at +FROM dispensaries d +LEFT JOIN dutchie_products dp ON d.id = dp.dispensary_id +WHERE d.menu_type = 'dutchie' +GROUP BY d.id, d.name, d.state; + +COMMENT ON VIEW v_visibility_summary IS 'Aggregated visibility metrics per dispensary for dashboard analytics'; + +-- ============================================================ +-- 4. RECORD MIGRATION +-- ============================================================ + +INSERT INTO schema_migrations (version, name, applied_at) +VALUES (57, '057_visibility_tracking_columns', NOW()) +ON CONFLICT (version) DO NOTHING; diff --git a/backend/migrations/058_add_id_resolution_columns.sql b/backend/migrations/058_add_id_resolution_columns.sql new file mode 100644 index 00000000..2ad57e19 --- /dev/null +++ b/backend/migrations/058_add_id_resolution_columns.sql @@ -0,0 +1,46 @@ +-- Migration 058: Add ID resolution tracking columns to dispensaries +-- +-- Supports Henry (Entry Point Finder) worker tracking: +-- - id_resolution_attempts: Count of how many times we've tried to resolve platform ID +-- - last_id_resolution_at: When we last tried (matches code expectation) +-- - id_resolution_status: Current status (pending, resolved, failed) +-- - id_resolution_error: Last error message from resolution attempt + +-- ============================================================ +-- 1. ADD ID RESOLUTION COLUMNS TO dispensaries +-- ============================================================ + +ALTER TABLE dispensaries + ADD COLUMN IF NOT EXISTS id_resolution_attempts INTEGER DEFAULT 0, + ADD COLUMN IF NOT EXISTS last_id_resolution_at TIMESTAMPTZ, + ADD COLUMN IF NOT EXISTS id_resolution_status VARCHAR(20) DEFAULT 'pending', + ADD COLUMN IF NOT EXISTS id_resolution_error TEXT; + +COMMENT ON COLUMN dispensaries.id_resolution_attempts IS 'Number of attempts to resolve platform_dispensary_id'; +COMMENT ON COLUMN dispensaries.last_id_resolution_at IS 'Timestamp of last ID resolution attempt'; +COMMENT ON COLUMN dispensaries.id_resolution_status IS 'Status: pending, resolved, failed'; +COMMENT ON COLUMN dispensaries.id_resolution_error IS 'Last error message from ID resolution attempt'; + +-- Additional columns needed by worker/scheduler +ALTER TABLE dispensaries + ADD COLUMN IF NOT EXISTS failed_at TIMESTAMPTZ, + ADD COLUMN IF NOT EXISTS failure_notes TEXT; + +COMMENT ON COLUMN dispensaries.failed_at IS 'Timestamp when dispensary was marked as permanently failed'; +COMMENT ON COLUMN dispensaries.failure_notes IS 'Notes about why dispensary was marked as failed'; + +-- ============================================================ +-- 2. CREATE INDEX FOR RESOLUTION QUERIES +-- ============================================================ + +CREATE INDEX IF NOT EXISTS idx_dispensaries_id_resolution_status + ON dispensaries(id_resolution_status) + WHERE id_resolution_status = 'pending'; + +-- ============================================================ +-- 3. RECORD MIGRATION +-- ============================================================ + +INSERT INTO schema_migrations (version, name, applied_at) +VALUES (58, '058_add_id_resolution_columns', NOW()) +ON CONFLICT (version) DO NOTHING; diff --git a/backend/migrations/059_job_queue_columns.sql b/backend/migrations/059_job_queue_columns.sql new file mode 100644 index 00000000..a1dca575 --- /dev/null +++ b/backend/migrations/059_job_queue_columns.sql @@ -0,0 +1,67 @@ +-- Migration 059: Add missing columns to dispensary_crawl_jobs +-- +-- Required for worker job processing: +-- - max_retries: Maximum retry attempts for a job +-- - retry_count: Current retry count +-- - worker_id: ID of worker processing the job +-- - locked_at: When the job was locked by a worker +-- - locked_by: Hostname of worker that locked the job + +-- ============================================================ +-- 1. ADD JOB QUEUE COLUMNS +-- ============================================================ + +ALTER TABLE dispensary_crawl_jobs + ADD COLUMN IF NOT EXISTS max_retries INTEGER DEFAULT 3, + ADD COLUMN IF NOT EXISTS retry_count INTEGER DEFAULT 0, + ADD COLUMN IF NOT EXISTS worker_id VARCHAR(100), + ADD COLUMN IF NOT EXISTS locked_at TIMESTAMPTZ, + ADD COLUMN IF NOT EXISTS locked_by VARCHAR(100), + ADD COLUMN IF NOT EXISTS updated_at TIMESTAMPTZ DEFAULT NOW(); + +COMMENT ON COLUMN dispensary_crawl_jobs.max_retries IS 'Maximum number of retry attempts'; +COMMENT ON COLUMN dispensary_crawl_jobs.retry_count IS 'Current retry count'; +COMMENT ON COLUMN dispensary_crawl_jobs.worker_id IS 'ID of worker processing this job'; +COMMENT ON COLUMN dispensary_crawl_jobs.locked_at IS 'When job was locked by worker'; +COMMENT ON COLUMN dispensary_crawl_jobs.locked_by IS 'Hostname of worker that locked job'; + +-- ============================================================ +-- 2. CREATE INDEXES FOR JOB QUEUE QUERIES +-- ============================================================ + +CREATE INDEX IF NOT EXISTS idx_crawl_jobs_status_priority + ON dispensary_crawl_jobs(status, priority DESC) + WHERE status = 'pending'; + +CREATE INDEX IF NOT EXISTS idx_crawl_jobs_worker_id + ON dispensary_crawl_jobs(worker_id) + WHERE worker_id IS NOT NULL; + +CREATE INDEX IF NOT EXISTS idx_crawl_jobs_locked_at + ON dispensary_crawl_jobs(locked_at) + WHERE locked_at IS NOT NULL; + +-- ============================================================ +-- 3. CREATE QUEUE STATS VIEW +-- ============================================================ + +CREATE OR REPLACE VIEW v_queue_stats AS +SELECT + COUNT(*) FILTER (WHERE status = 'pending') AS pending_jobs, + COUNT(*) FILTER (WHERE status = 'running') AS running_jobs, + COUNT(*) FILTER (WHERE status = 'completed' AND completed_at > NOW() - INTERVAL '1 hour') AS completed_1h, + COUNT(*) FILTER (WHERE status = 'failed' AND completed_at > NOW() - INTERVAL '1 hour') AS failed_1h, + COUNT(DISTINCT worker_id) FILTER (WHERE status = 'running') AS active_workers, + ROUND((AVG(EXTRACT(EPOCH FROM (completed_at - started_at))) + FILTER (WHERE status = 'completed' AND completed_at > NOW() - INTERVAL '1 hour'))::numeric, 2) AS avg_duration_seconds +FROM dispensary_crawl_jobs; + +COMMENT ON VIEW v_queue_stats IS 'Real-time queue statistics for monitoring dashboard'; + +-- ============================================================ +-- 4. RECORD MIGRATION +-- ============================================================ + +INSERT INTO schema_migrations (version, name, applied_at) +VALUES (59, '059_job_queue_columns', NOW()) +ON CONFLICT (version) DO NOTHING; diff --git a/backend/src/db/pool.ts b/backend/src/db/pool.ts new file mode 100644 index 00000000..cdc64472 --- /dev/null +++ b/backend/src/db/pool.ts @@ -0,0 +1,94 @@ +/** + * Runtime Database Pool + * + * This is the canonical database pool for all runtime services. + * Import pool from here, NOT from migrate.ts. + * + * migrate.ts is for CLI migrations only and must NOT be imported at runtime. + */ + +import dotenv from 'dotenv'; +import { Pool } from 'pg'; + +// Load .env before any env var access +dotenv.config(); + +/** + * Get the database connection string from environment variables. + * Supports both CANNAIQ_DB_URL and individual CANNAIQ_DB_* vars. + */ +function getConnectionString(): string { + // Priority 1: Full connection URL + if (process.env.CANNAIQ_DB_URL) { + return process.env.CANNAIQ_DB_URL; + } + + // Priority 2: Build from individual env vars + const host = process.env.CANNAIQ_DB_HOST; + const port = process.env.CANNAIQ_DB_PORT; + const name = process.env.CANNAIQ_DB_NAME; + const user = process.env.CANNAIQ_DB_USER; + const pass = process.env.CANNAIQ_DB_PASS; + + // Check if all individual vars are present + if (host && port && name && user && pass) { + return `postgresql://${user}:${pass}@${host}:${port}/${name}`; + } + + // Fallback: Try DATABASE_URL for legacy compatibility + if (process.env.DATABASE_URL) { + return process.env.DATABASE_URL; + } + + // Report what's missing + const required = ['CANNAIQ_DB_HOST', 'CANNAIQ_DB_PORT', 'CANNAIQ_DB_NAME', 'CANNAIQ_DB_USER', 'CANNAIQ_DB_PASS']; + const missing = required.filter((key) => !process.env[key]); + + throw new Error( + `[DB Pool] Missing database configuration.\n` + + `Set CANNAIQ_DB_URL, or all of: ${missing.join(', ')}` + ); +} + +// Lazy-initialized pool singleton +let _pool: Pool | null = null; + +/** + * Get the database pool (lazy singleton) + */ +export function getPool(): Pool { + if (!_pool) { + _pool = new Pool({ + connectionString: getConnectionString(), + max: 10, + idleTimeoutMillis: 30000, + connectionTimeoutMillis: 5000, + }); + + _pool.on('error', (err) => { + console.error('[DB Pool] Unexpected error on idle client:', err); + }); + } + return _pool; +} + +/** + * The database pool for runtime use. + * This is a getter that lazily initializes on first access. + */ +export const pool = { + query: (...args: Parameters) => getPool().query(...args), + connect: () => getPool().connect(), + end: () => getPool().end(), + on: (event: 'error' | 'connect' | 'acquire' | 'remove' | 'release', listener: (...args: any[]) => void) => getPool().on(event as any, listener), +}; + +/** + * Close the pool connection + */ +export async function closePool(): Promise { + if (_pool) { + await _pool.end(); + _pool = null; + } +} diff --git a/backend/src/dutchie-az/services/job-queue.ts b/backend/src/dutchie-az/services/job-queue.ts index 70ec3a23..d2908b30 100644 --- a/backend/src/dutchie-az/services/job-queue.ts +++ b/backend/src/dutchie-az/services/job-queue.ts @@ -8,6 +8,10 @@ import { query, getClient } from '../db/connection'; import { v4 as uuidv4 } from 'uuid'; import * as os from 'os'; +import { DEFAULT_CONFIG } from './store-validator'; + +// Minimum gap between crawls for the same dispensary (in minutes) +const MIN_CRAWL_GAP_MINUTES = DEFAULT_CONFIG.minCrawlGapMinutes; // 2 minutes // ============================================================ // TYPES @@ -97,11 +101,30 @@ export function getWorkerHostname(): string { // JOB ENQUEUEING // ============================================================ +export interface EnqueueResult { + jobId: number | null; + skipped: boolean; + reason?: 'already_queued' | 'too_soon' | 'error'; + message?: string; +} + /** * Enqueue a new job for processing * Returns null if a pending/running job already exists for this dispensary + * or if a job was completed/failed within the minimum gap period */ export async function enqueueJob(options: EnqueueJobOptions): Promise { + const result = await enqueueJobWithReason(options); + return result.jobId; +} + +/** + * Enqueue a new job with detailed result info + * Enforces: + * 1. No duplicate pending/running jobs for same dispensary + * 2. Minimum 2-minute gap between crawls for same dispensary + */ +export async function enqueueJobWithReason(options: EnqueueJobOptions): Promise { const { jobType, dispensaryId, @@ -121,31 +144,87 @@ export async function enqueueJob(options: EnqueueJobOptions): Promise 0) { console.log(`[JobQueue] Skipping enqueue - job already exists for dispensary ${dispensaryId}`); - return null; + return { + jobId: null, + skipped: true, + reason: 'already_queued', + message: `Job already pending/running for dispensary ${dispensaryId}`, + }; + } + + // Check minimum gap since last job (2 minutes) + const { rows: recent } = await query( + `SELECT id, created_at, status + FROM dispensary_crawl_jobs + WHERE dispensary_id = $1 + ORDER BY created_at DESC + LIMIT 1`, + [dispensaryId] + ); + + if (recent.length > 0) { + const lastJobTime = new Date(recent[0].created_at); + const minGapMs = MIN_CRAWL_GAP_MINUTES * 60 * 1000; + const timeSinceLastJob = Date.now() - lastJobTime.getTime(); + + if (timeSinceLastJob < minGapMs) { + const waitSeconds = Math.ceil((minGapMs - timeSinceLastJob) / 1000); + console.log(`[JobQueue] Skipping enqueue - minimum ${MIN_CRAWL_GAP_MINUTES}min gap not met for dispensary ${dispensaryId}. Wait ${waitSeconds}s`); + return { + jobId: null, + skipped: true, + reason: 'too_soon', + message: `Minimum ${MIN_CRAWL_GAP_MINUTES}-minute gap required. Try again in ${waitSeconds} seconds.`, + }; + } } } - const { rows } = await query( - `INSERT INTO dispensary_crawl_jobs (job_type, dispensary_id, status, priority, max_retries, metadata, created_at) - VALUES ($1, $2, 'pending', $3, $4, $5, NOW()) - RETURNING id`, - [jobType, dispensaryId || null, priority, maxRetries, metadata ? JSON.stringify(metadata) : null] - ); + try { + const { rows } = await query( + `INSERT INTO dispensary_crawl_jobs (job_type, dispensary_id, status, priority, max_retries, metadata, created_at) + VALUES ($1, $2, 'pending', $3, $4, $5, NOW()) + RETURNING id`, + [jobType, dispensaryId || null, priority, maxRetries, metadata ? JSON.stringify(metadata) : null] + ); - const jobId = rows[0].id; - console.log(`[JobQueue] Enqueued job ${jobId} (type=${jobType}, dispensary=${dispensaryId})`); - return jobId; + const jobId = rows[0].id; + console.log(`[JobQueue] Enqueued job ${jobId} (type=${jobType}, dispensary=${dispensaryId})`); + return { jobId, skipped: false }; + } catch (error: any) { + // Handle database trigger rejection for minimum gap + if (error.message?.includes('Minimum') && error.message?.includes('gap')) { + console.log(`[JobQueue] DB rejected - minimum gap not met for dispensary ${dispensaryId}`); + return { + jobId: null, + skipped: true, + reason: 'too_soon', + message: error.message, + }; + } + throw error; + } +} + +export interface BulkEnqueueResult { + enqueued: number; + skipped: number; + skippedReasons: { + alreadyQueued: number; + tooSoon: number; + }; } /** * Bulk enqueue jobs for multiple dispensaries * Skips dispensaries that already have pending/running jobs + * or have jobs within the minimum gap period */ export async function bulkEnqueueJobs( jobType: string, dispensaryIds: number[], options: { priority?: number; metadata?: Record } = {} -): Promise<{ enqueued: number; skipped: number }> { +): Promise { const { priority = 0, metadata } = options; // Get dispensaries that already have pending/running jobs @@ -156,11 +235,31 @@ export async function bulkEnqueueJobs( ); const existingSet = new Set(existing.map((r: any) => r.dispensary_id)); - // Filter out dispensaries with existing jobs - const toEnqueue = dispensaryIds.filter(id => !existingSet.has(id)); + // Get dispensaries that have recent jobs within minimum gap + const { rows: recent } = await query( + `SELECT DISTINCT dispensary_id FROM dispensary_crawl_jobs + WHERE dispensary_id = ANY($1) + AND created_at > NOW() - ($2 || ' minutes')::INTERVAL + AND dispensary_id NOT IN ( + SELECT dispensary_id FROM dispensary_crawl_jobs + WHERE dispensary_id = ANY($1) AND status IN ('pending', 'running') + )`, + [dispensaryIds, MIN_CRAWL_GAP_MINUTES] + ); + const recentSet = new Set(recent.map((r: any) => r.dispensary_id)); + + // Filter out dispensaries with existing or recent jobs + const toEnqueue = dispensaryIds.filter(id => !existingSet.has(id) && !recentSet.has(id)); if (toEnqueue.length === 0) { - return { enqueued: 0, skipped: dispensaryIds.length }; + return { + enqueued: 0, + skipped: dispensaryIds.length, + skippedReasons: { + alreadyQueued: existingSet.size, + tooSoon: recentSet.size, + }, + }; } // Bulk insert - each row needs 4 params: job_type, dispensary_id, priority, metadata @@ -181,8 +280,15 @@ export async function bulkEnqueueJobs( params ); - console.log(`[JobQueue] Bulk enqueued ${toEnqueue.length} jobs, skipped ${existingSet.size}`); - return { enqueued: toEnqueue.length, skipped: existingSet.size }; + console.log(`[JobQueue] Bulk enqueued ${toEnqueue.length} jobs, skipped ${existingSet.size} (queued) + ${recentSet.size} (recent)`); + return { + enqueued: toEnqueue.length, + skipped: existingSet.size + recentSet.size, + skippedReasons: { + alreadyQueued: existingSet.size, + tooSoon: recentSet.size, + }, + }; } // ============================================================ @@ -311,22 +417,48 @@ export async function heartbeat(jobId: number): Promise { /** * Mark job as completed + * + * Stores visibility tracking stats (visibilityLostCount, visibilityRestoredCount) + * in the metadata JSONB column for dashboard analytics. */ export async function completeJob( jobId: number, - result: { productsFound?: number; productsUpserted?: number; snapshotsCreated?: number } + result: { + productsFound?: number; + productsUpserted?: number; + snapshotsCreated?: number; + visibilityLostCount?: number; + visibilityRestoredCount?: number; + } ): Promise { + // Build metadata with visibility stats if provided + const metadata: Record = {}; + if (result.visibilityLostCount !== undefined) { + metadata.visibilityLostCount = result.visibilityLostCount; + } + if (result.visibilityRestoredCount !== undefined) { + metadata.visibilityRestoredCount = result.visibilityRestoredCount; + } + if (result.snapshotsCreated !== undefined) { + metadata.snapshotsCreated = result.snapshotsCreated; + } + await query( `UPDATE dispensary_crawl_jobs SET status = 'completed', completed_at = NOW(), products_found = COALESCE($2, products_found), - products_upserted = COALESCE($3, products_upserted), - snapshots_created = COALESCE($4, snapshots_created), + products_updated = COALESCE($3, products_updated), + metadata = COALESCE(metadata, '{}'::jsonb) || $4::jsonb, updated_at = NOW() WHERE id = $1`, - [jobId, result.productsFound, result.productsUpserted, result.snapshotsCreated] + [ + jobId, + result.productsFound, + result.productsUpserted, + JSON.stringify(metadata), + ] ); console.log(`[JobQueue] Job ${jobId} completed`); } diff --git a/backend/src/dutchie-az/services/product-crawler.ts b/backend/src/dutchie-az/services/product-crawler.ts index 976a0474..9b48c246 100644 --- a/backend/src/dutchie-az/services/product-crawler.ts +++ b/backend/src/dutchie-az/services/product-crawler.ts @@ -24,12 +24,8 @@ import { } from '../types'; import { downloadProductImage, imageExists } from '../../utils/image-storage'; -// Explicit column list for dispensaries table (avoids SELECT * issues with schema differences) -const DISPENSARY_COLUMNS = ` - id, name, slug, city, state, zip, address, latitude, longitude, - menu_type, menu_url, platform_dispensary_id, website, - provider_detection_data, created_at, updated_at -`; +// Use shared dispensary columns (handles optional columns like provider_detection_data) +import { DISPENSARY_COLUMNS } from '../db/dispensary-columns'; // ============================================================ // BATCH PROCESSING CONFIGURATION @@ -648,10 +644,15 @@ async function updateDispensaryCrawlStats( } /** - * Mark products as missing from feed + * Mark products as missing from feed (visibility-loss detection) * Creates a snapshot with isPresentInFeed=false and stockStatus='missing_from_feed' * for products that were NOT in the UNION of Mode A and Mode B product lists * + * Bella (Product Sync) visibility tracking: + * - Sets visibility_lost=TRUE and visibility_lost_at=NOW() for disappearing products + * - Records visibility event in snapshot metadata JSONB + * - NEVER deletes products, just marks them as visibility-lost + * * IMPORTANT: Uses UNION of both modes to avoid false positives * If the union is empty (possible outage), we skip marking to avoid data corruption */ @@ -660,25 +661,28 @@ async function markMissingProducts( platformDispensaryId: string, modeAProductIds: Set, modeBProductIds: Set, - pricingType: 'rec' | 'med' -): Promise { + pricingType: 'rec' | 'med', + workerName: string = 'Bella' +): Promise<{ markedMissing: number; newlyLost: number }> { // Build UNION of Mode A + Mode B product IDs const unionProductIds = new Set([...Array.from(modeAProductIds), ...Array.from(modeBProductIds)]); // OUTAGE DETECTION: If union is empty, something went wrong - don't mark anything as missing if (unionProductIds.size === 0) { - console.warn('[ProductCrawler] OUTAGE DETECTED: Both Mode A and Mode B returned 0 products. Skipping missing product marking.'); - return 0; + console.warn(`[${workerName} - Product Sync] OUTAGE DETECTED: Both Mode A and Mode B returned 0 products. Skipping visibility-loss marking.`); + return { markedMissing: 0, newlyLost: 0 }; } // Get all existing products for this dispensary that were not in the UNION + // Also check if they were already marked as visibility_lost to track new losses const { rows: missingProducts } = await query<{ id: number; external_product_id: string; name: string; + visibility_lost: boolean; }>( ` - SELECT id, external_product_id, name + SELECT id, external_product_id, name, COALESCE(visibility_lost, FALSE) as visibility_lost FROM dutchie_products WHERE dispensary_id = $1 AND external_product_id NOT IN (SELECT unnest($2::text[])) @@ -687,59 +691,141 @@ async function markMissingProducts( ); if (missingProducts.length === 0) { - return 0; + return { markedMissing: 0, newlyLost: 0 }; } - console.log(`[ProductCrawler] Marking ${missingProducts.length} products as missing from feed (union of ${modeAProductIds.size} Mode A + ${modeBProductIds.size} Mode B = ${unionProductIds.size} unique)...`); + // Separate newly lost products from already-lost products + const newlyLostProducts = missingProducts.filter(p => !p.visibility_lost); + const alreadyLostProducts = missingProducts.filter(p => p.visibility_lost); + + console.log(`[${workerName} - Product Sync] Visibility check: ${missingProducts.length} products missing (${newlyLostProducts.length} newly lost, ${alreadyLostProducts.length} already lost)`); const crawledAt = new Date(); - // Build all missing snapshots first (per CLAUDE.md Rule #15 - batch writes) - const missingSnapshots: Partial[] = missingProducts.map(product => ({ - dutchieProductId: product.id, - dispensaryId, - platformDispensaryId, - externalProductId: product.external_product_id, - pricingType, - crawlMode: 'mode_a' as CrawlMode, // Use mode_a for missing snapshots (convention) - status: undefined, - featured: false, - special: false, - medicalOnly: false, - recOnly: false, - isPresentInFeed: false, - stockStatus: 'missing_from_feed' as StockStatus, - totalQuantityAvailable: undefined, // null = unknown, not 0 - manualInventory: false, - isBelowThreshold: false, - isBelowKioskThreshold: false, - options: [], - rawPayload: { _missingFromFeed: true, lastKnownName: product.name }, - crawledAt, - })); + // Build all missing snapshots with visibility_events metadata + const missingSnapshots: Partial[] = missingProducts.map(product => { + const isNewlyLost = !product.visibility_lost; + return { + dutchieProductId: product.id, + dispensaryId, + platformDispensaryId, + externalProductId: product.external_product_id, + pricingType, + crawlMode: 'mode_a' as CrawlMode, + status: undefined, + featured: false, + special: false, + medicalOnly: false, + recOnly: false, + isPresentInFeed: false, + stockStatus: 'missing_from_feed' as StockStatus, + totalQuantityAvailable: undefined, + manualInventory: false, + isBelowThreshold: false, + isBelowKioskThreshold: false, + options: [], + rawPayload: { + _missingFromFeed: true, + lastKnownName: product.name, + visibility_events: isNewlyLost ? [{ + event_type: 'visibility_lost', + timestamp: crawledAt.toISOString(), + worker_name: workerName, + }] : [], + }, + crawledAt, + }; + }); // Batch insert missing snapshots const snapshotsInserted = await batchInsertSnapshots(missingSnapshots); - // Batch update product stock status in chunks + // Batch update product visibility status in chunks const productIds = missingProducts.map(p => p.id); const productChunks = chunkArray(productIds, BATCH_CHUNK_SIZE); - console.log(`[ProductCrawler] Updating ${productIds.length} product statuses in ${productChunks.length} chunks...`); + console.log(`[${workerName} - Product Sync] Updating ${productIds.length} product visibility in ${productChunks.length} chunks...`); for (const chunk of productChunks) { + // Update all products: set stock_status to missing + // Only set visibility_lost_at for NEWLY lost products (not already lost) await query( ` UPDATE dutchie_products - SET stock_status = 'missing_from_feed', total_quantity_available = NULL, updated_at = NOW() + SET + stock_status = 'missing_from_feed', + total_quantity_available = NULL, + visibility_lost = TRUE, + visibility_lost_at = CASE + WHEN visibility_lost IS NULL OR visibility_lost = FALSE THEN NOW() + ELSE visibility_lost_at -- Keep existing timestamp for already-lost products + END, + updated_at = NOW() WHERE id = ANY($1::int[]) `, [chunk] ); } - console.log(`[ProductCrawler] Marked ${snapshotsInserted} products as missing from feed`); - return snapshotsInserted; + console.log(`[${workerName} - Product Sync] Marked ${snapshotsInserted} products as missing, ${newlyLostProducts.length} newly visibility-lost`); + return { markedMissing: snapshotsInserted, newlyLost: newlyLostProducts.length }; +} + +/** + * Restore visibility for products that reappeared in the feed + * Called when products that were previously visibility_lost=TRUE are now found in the feed + * + * Bella (Product Sync) visibility tracking: + * - Sets visibility_lost=FALSE and visibility_restored_at=NOW() + * - Logs the restoration event + */ +async function restoreVisibilityForProducts( + dispensaryId: number, + productIds: Set, + workerName: string = 'Bella' +): Promise { + if (productIds.size === 0) { + return 0; + } + + // Find products that were visibility_lost and are now in the feed + const { rows: restoredProducts } = await query<{ id: number; external_product_id: string }>( + ` + SELECT id, external_product_id + FROM dutchie_products + WHERE dispensary_id = $1 + AND visibility_lost = TRUE + AND external_product_id = ANY($2::text[]) + `, + [dispensaryId, Array.from(productIds)] + ); + + if (restoredProducts.length === 0) { + return 0; + } + + console.log(`[${workerName} - Product Sync] Restoring visibility for ${restoredProducts.length} products that reappeared`); + + // Batch update restored products + const restoredIds = restoredProducts.map(p => p.id); + const chunks = chunkArray(restoredIds, BATCH_CHUNK_SIZE); + + for (const chunk of chunks) { + await query( + ` + UPDATE dutchie_products + SET + visibility_lost = FALSE, + visibility_restored_at = NOW(), + updated_at = NOW() + WHERE id = ANY($1::int[]) + `, + [chunk] + ); + } + + console.log(`[${workerName} - Product Sync] Restored visibility for ${restoredProducts.length} products`); + return restoredProducts.length; } // ============================================================ @@ -756,9 +842,12 @@ export interface CrawlResult { modeAProducts?: number; modeBProducts?: number; missingProductsMarked?: number; + visibilityLostCount?: number; // Products newly marked as visibility_lost + visibilityRestoredCount?: number; // Products restored from visibility_lost imagesDownloaded?: number; imageErrors?: number; errorMessage?: string; + httpStatus?: number; // HTTP status code for error classification durationMs: number; } @@ -1005,21 +1094,38 @@ export async function crawlDispensaryProducts( } } + // Build union of all product IDs found in both modes + const allFoundProductIds = new Set([ + ...Array.from(modeAProductIds), + ...Array.from(modeBProductIds), + ]); + + // VISIBILITY RESTORATION: Check if any previously-lost products have reappeared + const visibilityRestored = await restoreVisibilityForProducts( + dispensary.id, + allFoundProductIds, + 'Bella' + ); + // Mark products as missing using UNION of Mode A + Mode B // The function handles outage detection (empty union = skip marking) - missingMarked = await markMissingProducts( + // Now also tracks newly lost products vs already-lost products + const missingResult = await markMissingProducts( dispensary.id, dispensary.platformDispensaryId, modeAProductIds, modeBProductIds, - pricingType + pricingType, + 'Bella' ); + missingMarked = missingResult.markedMissing; + const newlyLostCount = missingResult.newlyLost; totalSnapshots += missingMarked; // Update dispensary stats await updateDispensaryCrawlStats(dispensary.id, totalUpserted); - console.log(`[ProductCrawler] Completed: ${totalUpserted} products, ${totalSnapshots} snapshots, ${missingMarked} marked missing, ${totalImagesDownloaded} images downloaded`); + console.log(`[Bella - Product Sync] Completed: ${totalUpserted} products, ${totalSnapshots} snapshots, ${missingMarked} missing, ${newlyLostCount} newly lost, ${visibilityRestored} restored, ${totalImagesDownloaded} images`); const totalProductsFound = modeAProducts + modeBProducts; return { @@ -1032,6 +1138,8 @@ export async function crawlDispensaryProducts( modeAProducts, modeBProducts, missingProductsMarked: missingMarked, + visibilityLostCount: newlyLostCount, + visibilityRestoredCount: visibilityRestored, imagesDownloaded: totalImagesDownloaded, imageErrors: totalImageErrors, durationMs: Date.now() - startTime, diff --git a/backend/src/dutchie-az/services/worker.ts b/backend/src/dutchie-az/services/worker.ts index aa3706a7..9269e4c6 100644 --- a/backend/src/dutchie-az/services/worker.ts +++ b/backend/src/dutchie-az/services/worker.ts @@ -3,6 +3,8 @@ * * Polls the job queue and processes crawl jobs. * Each worker instance runs independently, claiming jobs atomically. + * + * Phase 1: Enhanced with self-healing logic, error taxonomy, and retry management. */ import { @@ -20,13 +22,36 @@ import { crawlDispensaryProducts } from './product-crawler'; import { mapDbRowToDispensary } from './discovery'; import { query } from '../db/connection'; -// Explicit column list for dispensaries table (avoids SELECT * issues with schema differences) -// NOTE: failed_at is included for worker compatibility checks -const DISPENSARY_COLUMNS = ` - id, name, slug, city, state, zip, address, latitude, longitude, - menu_type, menu_url, platform_dispensary_id, website, - provider_detection_data, created_at, updated_at, failed_at -`; +// Phase 1: Error taxonomy and retry management +import { + CrawlErrorCode, + CrawlErrorCodeType, + classifyError, + isRetryable, + shouldRotateProxy, + shouldRotateUserAgent, + createSuccessResult, + createFailureResult, + CrawlResult, +} from './error-taxonomy'; +import { + RetryManager, + RetryDecision, + calculateNextCrawlAt, + determineCrawlStatus, + shouldAttemptRecovery, + sleep, +} from './retry-manager'; +import { + CrawlRotator, + userAgentRotator, + updateDispensaryRotation, +} from './proxy-rotator'; +import { DEFAULT_CONFIG, validateStoreConfig, isCrawlable } from './store-validator'; + +// Use shared dispensary columns (handles optional columns like provider_detection_data) +// NOTE: Using WITH_FAILED variant for worker compatibility checks +import { DISPENSARY_COLUMNS_WITH_FAILED as DISPENSARY_COLUMNS } from '../db/dispensary-columns'; // ============================================================ // WORKER CONFIG @@ -236,66 +261,245 @@ async function processJob(job: QueuedJob): Promise { } } -// Maximum consecutive failures before flagging a dispensary -const MAX_CONSECUTIVE_FAILURES = 3; +// Thresholds for crawl status transitions +const DEGRADED_THRESHOLD = 3; // Mark as degraded after 3 consecutive failures +const FAILED_THRESHOLD = 10; // Mark as failed after 10 consecutive failures + +// For backwards compatibility +const MAX_CONSECUTIVE_FAILURES = FAILED_THRESHOLD; /** - * Record a successful crawl - resets failure counter + * Record a successful crawl - resets failure counter and restores active status */ -async function recordCrawlSuccess(dispensaryId: number): Promise { +async function recordCrawlSuccess( + dispensaryId: number, + result: CrawlResult +): Promise { + // Calculate next crawl time (use store's frequency or default) + const { rows: storeRows } = await query( + `SELECT crawl_frequency_minutes FROM dispensaries WHERE id = $1`, + [dispensaryId] + ); + const frequencyMinutes = storeRows[0]?.crawl_frequency_minutes || DEFAULT_CONFIG.crawlFrequencyMinutes; + const nextCrawlAt = calculateNextCrawlAt(0, frequencyMinutes); + + // Reset failure state and schedule next crawl await query( `UPDATE dispensaries SET consecutive_failures = 0, + crawl_status = 'active', + backoff_multiplier = 1.0, last_crawl_at = NOW(), + last_success_at = NOW(), + last_error_code = NULL, + next_crawl_at = $2, + total_attempts = COALESCE(total_attempts, 0) + 1, + total_successes = COALESCE(total_successes, 0) + 1, updated_at = NOW() WHERE id = $1`, - [dispensaryId] + [dispensaryId, nextCrawlAt] ); + + // Log to crawl_attempts table for analytics + await logCrawlAttempt(dispensaryId, result); + + console.log(`[Worker] Dispensary ${dispensaryId} crawl success. Next crawl at ${nextCrawlAt.toISOString()}`); } /** - * Record a crawl failure - increments counter and may flag dispensary - * Returns true if dispensary was flagged as failed + * Record a crawl failure with self-healing logic + * - Rotates proxy/UA based on error type + * - Transitions through: active -> degraded -> failed + * - Calculates backoff for next attempt */ -async function recordCrawlFailure(dispensaryId: number, errorMessage: string): Promise { - // Increment failure counter - const { rows } = await query( - `UPDATE dispensaries - SET consecutive_failures = consecutive_failures + 1, - last_failure_at = NOW(), - last_failure_reason = $2, - updated_at = NOW() - WHERE id = $1 - RETURNING consecutive_failures`, - [dispensaryId, errorMessage] +async function recordCrawlFailure( + dispensaryId: number, + errorMessage: string, + errorCode?: CrawlErrorCodeType, + httpStatus?: number, + context?: { + proxyUsed?: string; + userAgentUsed?: string; + attemptNumber?: number; + } +): Promise<{ wasFlagged: boolean; newStatus: string; shouldRotateProxy: boolean; shouldRotateUA: boolean }> { + // Classify the error if not provided + const code = errorCode || classifyError(errorMessage, httpStatus); + + // Get current state + const { rows: storeRows } = await query( + `SELECT + consecutive_failures, + crawl_status, + backoff_multiplier, + crawl_frequency_minutes, + current_proxy_id, + current_user_agent + FROM dispensaries WHERE id = $1`, + [dispensaryId] ); - const failures = rows[0]?.consecutive_failures || 0; - - // If we've hit the threshold, flag the dispensary as failed - if (failures >= MAX_CONSECUTIVE_FAILURES) { - await query( - `UPDATE dispensaries - SET failed_at = NOW(), - menu_type = NULL, - platform_dispensary_id = NULL, - failure_notes = $2, - updated_at = NOW() - WHERE id = $1`, - [dispensaryId, `Auto-flagged after ${failures} consecutive failures. Last error: ${errorMessage}`] - ); - console.log(`[Worker] Dispensary ${dispensaryId} flagged as FAILED after ${failures} consecutive failures`); - return true; + if (storeRows.length === 0) { + return { wasFlagged: false, newStatus: 'unknown', shouldRotateProxy: false, shouldRotateUA: false }; } - console.log(`[Worker] Dispensary ${dispensaryId} failure recorded (${failures}/${MAX_CONSECUTIVE_FAILURES})`); - return false; + const store = storeRows[0]; + const currentFailures = (store.consecutive_failures || 0) + 1; + const frequencyMinutes = store.crawl_frequency_minutes || DEFAULT_CONFIG.crawlFrequencyMinutes; + + // Determine if we should rotate proxy/UA based on error type + const rotateProxy = shouldRotateProxy(code); + const rotateUA = shouldRotateUserAgent(code); + + // Get new proxy/UA if rotation is needed + let newProxyId = store.current_proxy_id; + let newUserAgent = store.current_user_agent; + + if (rotateUA) { + newUserAgent = userAgentRotator.getNext(); + console.log(`[Worker] Rotating user agent for dispensary ${dispensaryId} after ${code}`); + } + + // Determine new crawl status + const newStatus = determineCrawlStatus(currentFailures, { + degraded: DEGRADED_THRESHOLD, + failed: FAILED_THRESHOLD, + }); + + // Calculate backoff multiplier and next crawl time + const newBackoffMultiplier = Math.min( + (store.backoff_multiplier || 1.0) * 1.5, + 4.0 // Max 4x backoff + ); + const nextCrawlAt = calculateNextCrawlAt(currentFailures, frequencyMinutes); + + // Update dispensary with new failure state + if (newStatus === 'failed') { + // Mark as failed - won't be crawled again until manual intervention + await query( + `UPDATE dispensaries + SET consecutive_failures = $2, + crawl_status = $3, + backoff_multiplier = $4, + last_failure_at = NOW(), + last_error_code = $5, + failed_at = NOW(), + failure_notes = $6, + next_crawl_at = NULL, + current_proxy_id = $7, + current_user_agent = $8, + total_attempts = COALESCE(total_attempts, 0) + 1, + updated_at = NOW() + WHERE id = $1`, + [ + dispensaryId, + currentFailures, + newStatus, + newBackoffMultiplier, + code, + `Auto-flagged after ${currentFailures} consecutive failures. Last error: ${errorMessage}`, + newProxyId, + newUserAgent, + ] + ); + console.log(`[Worker] Dispensary ${dispensaryId} marked as FAILED after ${currentFailures} failures (${code})`); + } else { + // Update failure count but keep crawling (active or degraded) + await query( + `UPDATE dispensaries + SET consecutive_failures = $2, + crawl_status = $3, + backoff_multiplier = $4, + last_failure_at = NOW(), + last_error_code = $5, + next_crawl_at = $6, + current_proxy_id = $7, + current_user_agent = $8, + total_attempts = COALESCE(total_attempts, 0) + 1, + updated_at = NOW() + WHERE id = $1`, + [ + dispensaryId, + currentFailures, + newStatus, + newBackoffMultiplier, + code, + nextCrawlAt, + newProxyId, + newUserAgent, + ] + ); + + if (newStatus === 'degraded') { + console.log(`[Worker] Dispensary ${dispensaryId} marked as DEGRADED (${currentFailures}/${FAILED_THRESHOLD} failures). Next crawl: ${nextCrawlAt.toISOString()}`); + } else { + console.log(`[Worker] Dispensary ${dispensaryId} failure recorded (${currentFailures}/${DEGRADED_THRESHOLD}). Next crawl: ${nextCrawlAt.toISOString()}`); + } + } + + // Log to crawl_attempts table + const result = createFailureResult( + dispensaryId, + new Date(), + errorMessage, + httpStatus, + context + ); + await logCrawlAttempt(dispensaryId, result); + + return { + wasFlagged: newStatus === 'failed', + newStatus, + shouldRotateProxy: rotateProxy, + shouldRotateUA: rotateUA, + }; +} + +/** + * Log a crawl attempt to the crawl_attempts table for analytics + */ +async function logCrawlAttempt( + dispensaryId: number, + result: CrawlResult +): Promise { + try { + await query( + `INSERT INTO crawl_attempts ( + dispensary_id, started_at, finished_at, duration_ms, + error_code, error_message, http_status, + attempt_number, proxy_used, user_agent_used, + products_found, products_upserted, snapshots_created, + created_at + ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, NOW())`, + [ + dispensaryId, + result.startedAt, + result.finishedAt, + result.durationMs, + result.errorCode, + result.errorMessage || null, + result.httpStatus || null, + result.attemptNumber, + result.proxyUsed || null, + result.userAgentUsed || null, + result.productsFound || 0, + result.productsUpserted || 0, + result.snapshotsCreated || 0, + ] + ); + } catch (error) { + // Don't fail the job if logging fails + console.error(`[Worker] Failed to log crawl attempt for dispensary ${dispensaryId}:`, error); + } } /** * Process a product crawl job for a single dispensary */ async function processProductCrawlJob(job: QueuedJob): Promise { + const startedAt = new Date(); + const userAgent = userAgentRotator.getCurrent(); + if (!job.dispensaryId) { throw new Error('Product crawl job requires dispensary_id'); } @@ -311,17 +515,35 @@ async function processProductCrawlJob(job: QueuedJob): Promise { } const dispensary = mapDbRowToDispensary(rows[0]); + const rawDispensary = rows[0]; // Check if dispensary is already flagged as failed - if (rows[0].failed_at) { + if (rawDispensary.failed_at) { console.log(`[Worker] Skipping dispensary ${job.dispensaryId} - already flagged as failed`); await completeJob(job.id, { productsFound: 0, productsUpserted: 0 }); return; } + // Check crawl status - skip if paused or failed + if (rawDispensary.crawl_status === 'paused' || rawDispensary.crawl_status === 'failed') { + console.log(`[Worker] Skipping dispensary ${job.dispensaryId} - crawl_status is ${rawDispensary.crawl_status}`); + await completeJob(job.id, { productsFound: 0, productsUpserted: 0 }); + return; + } + if (!dispensary.platformDispensaryId) { - // Record failure and potentially flag - await recordCrawlFailure(job.dispensaryId, 'Missing platform_dispensary_id'); + // Record failure with error taxonomy + const { wasFlagged } = await recordCrawlFailure( + job.dispensaryId, + 'Missing platform_dispensary_id', + CrawlErrorCode.MISSING_PLATFORM_ID, + undefined, + { userAgentUsed: userAgent, attemptNumber: job.retryCount + 1 } + ); + if (wasFlagged) { + await completeJob(job.id, { productsFound: 0, productsUpserted: 0 }); + return; + } throw new Error(`Dispensary ${job.dispensaryId} has no platform_dispensary_id`); } @@ -346,28 +568,67 @@ async function processProductCrawlJob(job: QueuedJob): Promise { }); if (result.success) { - // Success! Reset failure counter - await recordCrawlSuccess(job.dispensaryId); + // Success! Create result and record + const crawlResult = createSuccessResult( + job.dispensaryId, + startedAt, + { + productsFound: result.productsFetched, + productsUpserted: result.productsUpserted, + snapshotsCreated: result.snapshotsCreated, + }, + { + attemptNumber: job.retryCount + 1, + userAgentUsed: userAgent, + } + ); + await recordCrawlSuccess(job.dispensaryId, crawlResult); await completeJob(job.id, { productsFound: result.productsFetched, productsUpserted: result.productsUpserted, snapshotsCreated: result.snapshotsCreated, + // Visibility tracking stats for dashboard + visibilityLostCount: result.visibilityLostCount || 0, + visibilityRestoredCount: result.visibilityRestoredCount || 0, }); } else { - // Crawl returned failure - record it - const wasFlagged = await recordCrawlFailure(job.dispensaryId, result.errorMessage || 'Crawl failed'); + // Crawl returned failure - classify error and record + const errorCode = classifyError(result.errorMessage || 'Crawl failed', result.httpStatus); + const { wasFlagged } = await recordCrawlFailure( + job.dispensaryId, + result.errorMessage || 'Crawl failed', + errorCode, + result.httpStatus, + { userAgentUsed: userAgent, attemptNumber: job.retryCount + 1 } + ); + if (wasFlagged) { - // Don't throw - the dispensary is now flagged, job is "complete" + // Dispensary is now flagged - complete the job + await completeJob(job.id, { productsFound: 0, productsUpserted: 0 }); + } else if (!isRetryable(errorCode)) { + // Non-retryable error - complete as failed await completeJob(job.id, { productsFound: 0, productsUpserted: 0 }); } else { + // Retryable error - let job queue handle retry throw new Error(result.errorMessage || 'Crawl failed'); } } } catch (error: any) { - // Record the failure - const wasFlagged = await recordCrawlFailure(job.dispensaryId, error.message); + // Record the failure with error taxonomy + const errorCode = classifyError(error.message); + const { wasFlagged } = await recordCrawlFailure( + job.dispensaryId, + error.message, + errorCode, + undefined, + { userAgentUsed: userAgent, attemptNumber: job.retryCount + 1 } + ); + if (wasFlagged) { - // Dispensary is now flagged - complete the job rather than fail it + // Dispensary is now flagged - complete the job + await completeJob(job.id, { productsFound: 0, productsUpserted: 0 }); + } else if (!isRetryable(errorCode)) { + // Non-retryable error - complete as failed await completeJob(job.id, { productsFound: 0, productsUpserted: 0 }); } else { throw error; diff --git a/backend/src/multi-state/__tests__/state-query-service.test.ts b/backend/src/multi-state/__tests__/state-query-service.test.ts new file mode 100644 index 00000000..a973beb1 --- /dev/null +++ b/backend/src/multi-state/__tests__/state-query-service.test.ts @@ -0,0 +1,339 @@ +/** + * StateQueryService Unit Tests + * Phase 4: Multi-State Expansion + */ + +import { StateQueryService } from '../state-query-service'; + +// Mock the pool +const mockQuery = jest.fn(); +const mockPool = { + query: mockQuery, +} as any; + +describe('StateQueryService', () => { + let service: StateQueryService; + + beforeEach(() => { + jest.clearAllMocks(); + service = new StateQueryService(mockPool); + }); + + describe('listStates', () => { + it('should return all states ordered by name', async () => { + mockQuery.mockResolvedValueOnce({ + rows: [ + { code: 'AZ', name: 'Arizona' }, + { code: 'CA', name: 'California' }, + { code: 'NV', name: 'Nevada' }, + ], + }); + + const states = await service.listStates(); + + expect(states).toHaveLength(3); + expect(states[0].code).toBe('AZ'); + expect(mockQuery).toHaveBeenCalledWith(expect.stringContaining('FROM states')); + }); + }); + + describe('listActiveStates', () => { + it('should return only states with dispensary data', async () => { + mockQuery.mockResolvedValueOnce({ + rows: [ + { code: 'AZ', name: 'Arizona' }, + ], + }); + + const states = await service.listActiveStates(); + + expect(states).toHaveLength(1); + expect(states[0].code).toBe('AZ'); + expect(mockQuery).toHaveBeenCalledWith( + expect.stringContaining('JOIN dispensaries') + ); + }); + }); + + describe('getStateSummary', () => { + it('should return null for unknown state', async () => { + // Materialized view query returns empty + mockQuery.mockResolvedValueOnce({ rows: [] }); + + const summary = await service.getStateSummary('XX'); + + expect(summary).toBeNull(); + }); + + it('should return full summary for valid state', async () => { + // Materialized view query + mockQuery.mockResolvedValueOnce({ + rows: [{ + state: 'AZ', + stateName: 'Arizona', + storeCount: 100, + dutchieStores: 80, + activeStores: 75, + totalProducts: 5000, + inStockProducts: 4500, + onSpecialProducts: 200, + uniqueBrands: 150, + uniqueCategories: 10, + avgPriceRec: 45.50, + minPriceRec: 10.00, + maxPriceRec: 200.00, + refreshedAt: new Date(), + }], + }); + + // Crawl stats query + mockQuery.mockResolvedValueOnce({ + rows: [{ + recent_crawls: '50', + failed_crawls: '2', + last_crawl_at: new Date(), + }], + }); + + // Top brands + mockQuery.mockResolvedValueOnce({ + rows: [ + { brandId: 1, brandName: 'Brand 1', storeCount: 50, productCount: 100 }, + ], + }); + + // Top categories + mockQuery.mockResolvedValueOnce({ + rows: [ + { category: 'Flower', productCount: 1000, storeCount: 80 }, + ], + }); + + const summary = await service.getStateSummary('AZ'); + + expect(summary).not.toBeNull(); + expect(summary!.state).toBe('AZ'); + expect(summary!.storeCount).toBe(100); + expect(summary!.totalProducts).toBe(5000); + expect(summary!.recentCrawls).toBe(50); + expect(summary!.topBrands).toHaveLength(1); + expect(summary!.topCategories).toHaveLength(1); + }); + }); + + describe('getBrandsByState', () => { + it('should return brands for a state with pagination', async () => { + mockQuery.mockResolvedValueOnce({ + rows: [ + { + brandId: 1, + brandName: 'Brand A', + brandSlug: 'brand-a', + storeCount: 50, + productCount: 200, + avgPrice: 45.00, + firstSeenInState: new Date(), + lastSeenInState: new Date(), + }, + ], + }); + + const brands = await service.getBrandsByState('AZ', { limit: 10, offset: 0 }); + + expect(brands).toHaveLength(1); + expect(brands[0].brandName).toBe('Brand A'); + expect(mockQuery).toHaveBeenCalledWith( + expect.stringContaining('FROM v_brand_state_presence'), + ['AZ', 10, 0] + ); + }); + }); + + describe('getBrandStatePenetration', () => { + it('should return penetration data for a brand', async () => { + mockQuery.mockResolvedValueOnce({ + rows: [ + { + state: 'AZ', + stateName: 'Arizona', + totalStores: 100, + storesWithBrand: 50, + penetrationPct: 50.00, + productCount: 200, + avgPrice: 45.00, + }, + { + state: 'CA', + stateName: 'California', + totalStores: 200, + storesWithBrand: 60, + penetrationPct: 30.00, + productCount: 300, + avgPrice: 55.00, + }, + ], + }); + + const penetration = await service.getBrandStatePenetration(123); + + expect(penetration).toHaveLength(2); + expect(penetration[0].state).toBe('AZ'); + expect(penetration[0].penetrationPct).toBe(50.00); + expect(penetration[1].state).toBe('CA'); + }); + }); + + describe('compareBrandAcrossStates', () => { + it('should compare brand across specified states', async () => { + // Brand lookup + mockQuery.mockResolvedValueOnce({ + rows: [{ id: 1, name: 'Test Brand' }], + }); + + // Penetration data + mockQuery.mockResolvedValueOnce({ + rows: [ + { state: 'AZ', stateName: 'Arizona', penetrationPct: 50, totalStores: 100, storesWithBrand: 50, productCount: 200, avgPrice: 45 }, + { state: 'CA', stateName: 'California', penetrationPct: 30, totalStores: 200, storesWithBrand: 60, productCount: 300, avgPrice: 55 }, + ], + }); + + // National stats + mockQuery.mockResolvedValueOnce({ + rows: [{ total_stores: '300', stores_with_brand: '110', avg_price: 50 }], + }); + + const comparison = await service.compareBrandAcrossStates(1, ['AZ', 'CA']); + + expect(comparison.brandName).toBe('Test Brand'); + expect(comparison.states).toHaveLength(2); + expect(comparison.bestPerformingState).toBe('AZ'); + expect(comparison.worstPerformingState).toBe('CA'); + }); + + it('should throw error for unknown brand', async () => { + mockQuery.mockResolvedValueOnce({ rows: [] }); + + await expect(service.compareBrandAcrossStates(999, ['AZ'])).rejects.toThrow('Brand 999 not found'); + }); + }); + + describe('getCategoriesByState', () => { + it('should return categories for a state', async () => { + mockQuery.mockResolvedValueOnce({ + rows: [ + { category: 'Flower', productCount: 1000, storeCount: 80, avgPrice: 35, inStockCount: 900, onSpecialCount: 50 }, + { category: 'Edibles', productCount: 500, storeCount: 60, avgPrice: 25, inStockCount: 450, onSpecialCount: 30 }, + ], + }); + + const categories = await service.getCategoriesByState('AZ'); + + expect(categories).toHaveLength(2); + expect(categories[0].category).toBe('Flower'); + expect(categories[0].productCount).toBe(1000); + }); + }); + + describe('getStoresByState', () => { + it('should return stores with metrics', async () => { + mockQuery.mockResolvedValueOnce({ + rows: [ + { + dispensaryId: 1, + dispensaryName: 'Test Store', + dispensarySlug: 'test-store', + state: 'AZ', + city: 'Phoenix', + menuType: 'dutchie', + crawlStatus: 'active', + lastCrawlAt: new Date(), + productCount: 200, + inStockCount: 180, + brandCount: 50, + avgPrice: 45, + specialCount: 10, + }, + ], + }); + + const stores = await service.getStoresByState('AZ', { limit: 50 }); + + expect(stores).toHaveLength(1); + expect(stores[0].dispensaryName).toBe('Test Store'); + expect(stores[0].productCount).toBe(200); + }); + }); + + describe('getNationalSummary', () => { + it('should return national aggregate metrics', async () => { + // State metrics + mockQuery.mockResolvedValueOnce({ + rows: [ + { state: 'AZ', stateName: 'Arizona', storeCount: 100, totalProducts: 5000 }, + ], + }); + + // National counts + mockQuery.mockResolvedValueOnce({ + rows: [{ + total_states: '17', + active_states: '5', + total_stores: '500', + total_products: '25000', + total_brands: '300', + avg_price_national: 45.00, + }], + }); + + const summary = await service.getNationalSummary(); + + expect(summary.totalStates).toBe(17); + expect(summary.activeStates).toBe(5); + expect(summary.totalStores).toBe(500); + expect(summary.totalProducts).toBe(25000); + expect(summary.avgPriceNational).toBe(45.00); + }); + }); + + describe('getStateHeatmapData', () => { + it('should return heatmap data for stores metric', async () => { + mockQuery.mockResolvedValueOnce({ + rows: [ + { state: 'AZ', stateName: 'Arizona', value: 100, label: 'stores' }, + { state: 'CA', stateName: 'California', value: 200, label: 'stores' }, + ], + }); + + const heatmap = await service.getStateHeatmapData('stores'); + + expect(heatmap).toHaveLength(2); + expect(heatmap[0].state).toBe('AZ'); + expect(heatmap[0].value).toBe(100); + }); + + it('should require brandId for penetration metric', async () => { + await expect(service.getStateHeatmapData('penetration')).rejects.toThrow( + 'brandId required for penetration heatmap' + ); + }); + }); + + describe('isValidState', () => { + it('should return true for valid state code', async () => { + mockQuery.mockResolvedValueOnce({ rows: [{ code: 'AZ' }] }); + + const isValid = await service.isValidState('AZ'); + + expect(isValid).toBe(true); + }); + + it('should return false for invalid state code', async () => { + mockQuery.mockResolvedValueOnce({ rows: [] }); + + const isValid = await service.isValidState('XX'); + + expect(isValid).toBe(false); + }); + }); +}); diff --git a/backend/src/multi-state/index.ts b/backend/src/multi-state/index.ts new file mode 100644 index 00000000..bb06642c --- /dev/null +++ b/backend/src/multi-state/index.ts @@ -0,0 +1,15 @@ +/** + * Multi-State Module + * + * Central export for multi-state queries and analytics. + * Phase 4: Multi-State Expansion + */ + +// Types +export * from './types'; + +// Query Service +export { StateQueryService } from './state-query-service'; + +// Routes +export { createMultiStateRoutes } from './routes'; diff --git a/backend/src/multi-state/routes.ts b/backend/src/multi-state/routes.ts new file mode 100644 index 00000000..9285a5a7 --- /dev/null +++ b/backend/src/multi-state/routes.ts @@ -0,0 +1,451 @@ +/** + * Multi-State API Routes + * + * Endpoints for multi-state queries, analytics, and comparisons. + * Phase 4: Multi-State Expansion + */ + +import { Router, Request, Response } from 'express'; +import { Pool } from 'pg'; +import { StateQueryService } from './state-query-service'; +import { StateQueryOptions, CrossStateQueryOptions } from './types'; + +export function createMultiStateRoutes(pool: Pool): Router { + const router = Router(); + const stateService = new StateQueryService(pool); + + // ========================================================================= + // State List Endpoints + // ========================================================================= + + /** + * GET /api/states + * List all states (both configured and active) + */ + router.get('/states', async (req: Request, res: Response) => { + try { + const activeOnly = req.query.active === 'true'; + const states = activeOnly + ? await stateService.listActiveStates() + : await stateService.listStates(); + + res.json({ + success: true, + data: { + states, + count: states.length, + }, + }); + } catch (error: any) { + console.error('[MultiState] Error listing states:', error); + res.status(500).json({ success: false, error: error.message }); + } + }); + + // ========================================================================= + // State Summary Endpoints + // ========================================================================= + + /** + * GET /api/state/:state/summary + * Get detailed summary for a specific state + */ + router.get('/state/:state/summary', async (req: Request, res: Response) => { + try { + const { state } = req.params; + + // Validate state code + const isValid = await stateService.isValidState(state.toUpperCase()); + if (!isValid) { + return res.status(404).json({ + success: false, + error: `Unknown state: ${state}`, + }); + } + + const summary = await stateService.getStateSummary(state.toUpperCase()); + + if (!summary) { + return res.status(404).json({ + success: false, + error: `No data for state: ${state}`, + }); + } + + res.json({ + success: true, + data: summary, + }); + } catch (error: any) { + console.error(`[MultiState] Error getting state summary:`, error); + res.status(500).json({ success: false, error: error.message }); + } + }); + + /** + * GET /api/state/:state/brands + * Get brands in a specific state + */ + router.get('/state/:state/brands', async (req: Request, res: Response) => { + try { + const { state } = req.params; + const options: StateQueryOptions = { + limit: parseInt(req.query.limit as string) || 50, + offset: parseInt(req.query.offset as string) || 0, + sortBy: (req.query.sortBy as string) || 'productCount', + sortDir: (req.query.sortDir as 'asc' | 'desc') || 'desc', + }; + + const brands = await stateService.getBrandsByState(state.toUpperCase(), options); + + res.json({ + success: true, + data: { + state: state.toUpperCase(), + brands, + count: brands.length, + pagination: { + limit: options.limit, + offset: options.offset, + }, + }, + }); + } catch (error: any) { + console.error(`[MultiState] Error getting state brands:`, error); + res.status(500).json({ success: false, error: error.message }); + } + }); + + /** + * GET /api/state/:state/categories + * Get categories in a specific state + */ + router.get('/state/:state/categories', async (req: Request, res: Response) => { + try { + const { state } = req.params; + const options: StateQueryOptions = { + limit: parseInt(req.query.limit as string) || 50, + offset: parseInt(req.query.offset as string) || 0, + sortBy: (req.query.sortBy as string) || 'productCount', + sortDir: (req.query.sortDir as 'asc' | 'desc') || 'desc', + }; + + const categories = await stateService.getCategoriesByState(state.toUpperCase(), options); + + res.json({ + success: true, + data: { + state: state.toUpperCase(), + categories, + count: categories.length, + pagination: { + limit: options.limit, + offset: options.offset, + }, + }, + }); + } catch (error: any) { + console.error(`[MultiState] Error getting state categories:`, error); + res.status(500).json({ success: false, error: error.message }); + } + }); + + /** + * GET /api/state/:state/stores + * Get stores in a specific state + */ + router.get('/state/:state/stores', async (req: Request, res: Response) => { + try { + const { state } = req.params; + const options: StateQueryOptions = { + limit: parseInt(req.query.limit as string) || 100, + offset: parseInt(req.query.offset as string) || 0, + sortBy: (req.query.sortBy as string) || 'productCount', + sortDir: (req.query.sortDir as 'asc' | 'desc') || 'desc', + includeInactive: req.query.includeInactive === 'true', + }; + + const stores = await stateService.getStoresByState(state.toUpperCase(), options); + + res.json({ + success: true, + data: { + state: state.toUpperCase(), + stores, + count: stores.length, + pagination: { + limit: options.limit, + offset: options.offset, + }, + }, + }); + } catch (error: any) { + console.error(`[MultiState] Error getting state stores:`, error); + res.status(500).json({ success: false, error: error.message }); + } + }); + + /** + * GET /api/state/:state/analytics/prices + * Get price distribution for a state + */ + router.get('/state/:state/analytics/prices', async (req: Request, res: Response) => { + try { + const { state } = req.params; + const options = { + category: req.query.category as string | undefined, + brandId: req.query.brandId ? parseInt(req.query.brandId as string) : undefined, + }; + + const priceData = await stateService.getStorePriceDistribution( + state.toUpperCase(), + options + ); + + res.json({ + success: true, + data: { + state: state.toUpperCase(), + priceDistribution: priceData, + }, + }); + } catch (error: any) { + console.error(`[MultiState] Error getting price analytics:`, error); + res.status(500).json({ success: false, error: error.message }); + } + }); + + // ========================================================================= + // National Analytics Endpoints + // ========================================================================= + + /** + * GET /api/analytics/national/summary + * Get national summary across all states + */ + router.get('/analytics/national/summary', async (req: Request, res: Response) => { + try { + const summary = await stateService.getNationalSummary(); + + res.json({ + success: true, + data: summary, + }); + } catch (error: any) { + console.error(`[MultiState] Error getting national summary:`, error); + res.status(500).json({ success: false, error: error.message }); + } + }); + + /** + * GET /api/analytics/national/prices + * Get national price comparison across all states + */ + router.get('/analytics/national/prices', async (req: Request, res: Response) => { + try { + const options = { + category: req.query.category as string | undefined, + brandId: req.query.brandId ? parseInt(req.query.brandId as string) : undefined, + }; + + const priceData = await stateService.getNationalPriceComparison(options); + + res.json({ + success: true, + data: { + priceComparison: priceData, + filters: options, + }, + }); + } catch (error: any) { + console.error(`[MultiState] Error getting national prices:`, error); + res.status(500).json({ success: false, error: error.message }); + } + }); + + /** + * GET /api/analytics/national/heatmap + * GET /api/national/heatmap (alias) + * Get state heatmap data for various metrics + */ + const heatmapHandler = async (req: Request, res: Response) => { + try { + const metric = (req.query.metric as 'stores' | 'products' | 'brands' | 'avgPrice' | 'penetration') || 'stores'; + const brandId = req.query.brandId ? parseInt(req.query.brandId as string) : undefined; + const category = req.query.category as string | undefined; + + const heatmapData = await stateService.getStateHeatmapData(metric, { brandId, category }); + + res.json({ + success: true, + data: { + metric, + heatmap: heatmapData, + }, + }); + } catch (error: any) { + console.error(`[MultiState] Error getting heatmap data:`, error); + res.status(500).json({ success: false, error: error.message }); + } + }; + + // Register heatmap on both paths for compatibility + router.get('/analytics/national/heatmap', heatmapHandler); + router.get('/national/heatmap', heatmapHandler); + + /** + * GET /api/analytics/national/metrics + * Get all state metrics for dashboard + */ + router.get('/analytics/national/metrics', async (req: Request, res: Response) => { + try { + const metrics = await stateService.getAllStateMetrics(); + + res.json({ + success: true, + data: { + stateMetrics: metrics, + count: metrics.length, + }, + }); + } catch (error: any) { + console.error(`[MultiState] Error getting state metrics:`, error); + res.status(500).json({ success: false, error: error.message }); + } + }); + + // ========================================================================= + // Cross-State Comparison Endpoints + // ========================================================================= + + /** + * GET /api/analytics/compare/brand/:brandId + * Compare a brand across multiple states + */ + router.get('/analytics/compare/brand/:brandId', async (req: Request, res: Response) => { + try { + const brandId = parseInt(req.params.brandId); + const statesParam = req.query.states as string; + + // Parse states - either comma-separated or get all active states + let states: string[]; + if (statesParam) { + states = statesParam.split(',').map(s => s.trim().toUpperCase()); + } else { + const activeStates = await stateService.listActiveStates(); + states = activeStates.map(s => s.code); + } + + const comparison = await stateService.compareBrandAcrossStates(brandId, states); + + res.json({ + success: true, + data: comparison, + }); + } catch (error: any) { + console.error(`[MultiState] Error comparing brand across states:`, error); + res.status(500).json({ success: false, error: error.message }); + } + }); + + /** + * GET /api/analytics/compare/category/:category + * Compare a category across multiple states + */ + router.get('/analytics/compare/category/:category', async (req: Request, res: Response) => { + try { + const { category } = req.params; + const statesParam = req.query.states as string; + + // Parse states - either comma-separated or get all active states + let states: string[]; + if (statesParam) { + states = statesParam.split(',').map(s => s.trim().toUpperCase()); + } else { + const activeStates = await stateService.listActiveStates(); + states = activeStates.map(s => s.code); + } + + const comparison = await stateService.compareCategoryAcrossStates(category, states); + + res.json({ + success: true, + data: comparison, + }); + } catch (error: any) { + console.error(`[MultiState] Error comparing category across states:`, error); + res.status(500).json({ success: false, error: error.message }); + } + }); + + /** + * GET /api/analytics/brand/:brandId/penetration + * Get brand penetration across all states + */ + router.get('/analytics/brand/:brandId/penetration', async (req: Request, res: Response) => { + try { + const brandId = parseInt(req.params.brandId); + + const penetration = await stateService.getBrandStatePenetration(brandId); + + res.json({ + success: true, + data: { + brandId, + statePenetration: penetration, + }, + }); + } catch (error: any) { + console.error(`[MultiState] Error getting brand penetration:`, error); + res.status(500).json({ success: false, error: error.message }); + } + }); + + /** + * GET /api/analytics/brand/:brandId/trend + * Get national penetration trend for a brand + */ + router.get('/analytics/brand/:brandId/trend', async (req: Request, res: Response) => { + try { + const brandId = parseInt(req.params.brandId); + const days = parseInt(req.query.days as string) || 30; + + const trend = await stateService.getNationalPenetrationTrend(brandId, { days }); + + res.json({ + success: true, + data: trend, + }); + } catch (error: any) { + console.error(`[MultiState] Error getting brand trend:`, error); + res.status(500).json({ success: false, error: error.message }); + } + }); + + // ========================================================================= + // Admin Endpoints + // ========================================================================= + + /** + * POST /api/admin/states/refresh-metrics + * Manually refresh materialized views + */ + router.post('/admin/states/refresh-metrics', async (req: Request, res: Response) => { + try { + const startTime = Date.now(); + await stateService.refreshMetrics(); + const duration = Date.now() - startTime; + + res.json({ + success: true, + message: 'State metrics refreshed successfully', + durationMs: duration, + }); + } catch (error: any) { + console.error(`[MultiState] Error refreshing metrics:`, error); + res.status(500).json({ success: false, error: error.message }); + } + }); + + return router; +} diff --git a/backend/src/multi-state/state-query-service.ts b/backend/src/multi-state/state-query-service.ts new file mode 100644 index 00000000..1742a845 --- /dev/null +++ b/backend/src/multi-state/state-query-service.ts @@ -0,0 +1,643 @@ +/** + * StateQueryService + * + * Core service for multi-state queries and analytics. + * Phase 4: Multi-State Expansion + */ + +import { Pool } from 'pg'; +import { + State, + StateMetrics, + StateSummary, + BrandInState, + BrandStatePenetration, + BrandCrossStateComparison, + CategoryInState, + CategoryStateDist, + CategoryCrossStateComparison, + StoreInState, + StatePriceDistribution, + NationalSummary, + NationalPenetrationTrend, + StateHeatmapData, + StateQueryOptions, + CrossStateQueryOptions, +} from './types'; + +export class StateQueryService { + constructor(private pool: Pool) {} + + // ========================================================================= + // State List & Basic Queries + // ========================================================================= + + /** + * Get all available states + */ + async listStates(): Promise { + const result = await this.pool.query(` + SELECT code, name + FROM states + ORDER BY name + `); + return result.rows; + } + + /** + * Get states that have dispensary data + */ + async listActiveStates(): Promise { + const result = await this.pool.query(` + SELECT DISTINCT s.code, s.name + FROM states s + JOIN dispensaries d ON d.state = s.code + WHERE d.menu_type IS NOT NULL + ORDER BY s.name + `); + return result.rows; + } + + // ========================================================================= + // State Summary & Metrics + // ========================================================================= + + /** + * Get summary metrics for a single state + */ + async getStateSummary(state: string): Promise { + // Get base metrics from materialized view + const metricsResult = await this.pool.query(` + SELECT + state, + state_name AS "stateName", + dispensary_count AS "storeCount", + dispensary_count AS "dutchieStores", + dispensary_count AS "activeStores", + total_products AS "totalProducts", + in_stock_products AS "inStockProducts", + out_of_stock_products AS "outOfStockProducts", + unique_brands AS "uniqueBrands", + unique_categories AS "uniqueCategories", + avg_price_rec AS "avgPriceRec", + min_price_rec AS "minPriceRec", + max_price_rec AS "maxPriceRec", + refreshed_at AS "refreshedAt" + FROM mv_state_metrics + WHERE state = $1 + `, [state]); + + if (metricsResult.rows.length === 0) { + return null; + } + + const metrics = metricsResult.rows[0]; + + // Get crawl stats + const crawlResult = await this.pool.query(` + SELECT + COUNT(*) FILTER (WHERE cr.status = 'success' AND cr.started_at > NOW() - INTERVAL '24 hours') AS recent_crawls, + COUNT(*) FILTER (WHERE cr.status = 'failed' AND cr.started_at > NOW() - INTERVAL '24 hours') AS failed_crawls, + MAX(cr.finished_at) AS last_crawl_at + FROM crawl_runs cr + JOIN dispensaries d ON cr.dispensary_id = d.id + WHERE d.state = $1 + `, [state]); + + // Get top brands + const topBrands = await this.getBrandsByState(state, { limit: 5 }); + + // Get top categories + const topCategories = await this.getCategoriesByState(state, { limit: 5 }); + + return { + ...metrics, + recentCrawls: parseInt(crawlResult.rows[0]?.recent_crawls || '0'), + failedCrawls: parseInt(crawlResult.rows[0]?.failed_crawls || '0'), + lastCrawlAt: crawlResult.rows[0]?.last_crawl_at || null, + topBrands, + topCategories, + }; + } + + /** + * Get metrics for all states + */ + async getAllStateMetrics(): Promise { + const result = await this.pool.query(` + SELECT + state, + state_name AS "stateName", + dispensary_count AS "storeCount", + dispensary_count AS "dutchieStores", + dispensary_count AS "activeStores", + total_products AS "totalProducts", + in_stock_products AS "inStockProducts", + out_of_stock_products AS "outOfStockProducts", + unique_brands AS "uniqueBrands", + unique_categories AS "uniqueCategories", + avg_price_rec AS "avgPriceRec", + min_price_rec AS "minPriceRec", + max_price_rec AS "maxPriceRec", + refreshed_at AS "refreshedAt" + FROM mv_state_metrics + ORDER BY dispensary_count DESC + `); + return result.rows; + } + + // ========================================================================= + // Brand Queries + // ========================================================================= + + /** + * Get brands present in a specific state + */ + async getBrandsByState(state: string, options: StateQueryOptions = {}): Promise { + const { limit = 50, offset = 0, sortBy = 'productCount', sortDir = 'desc' } = options; + + const sortColumn = { + productCount: 'product_count', + storeCount: 'store_count', + avgPrice: 'avg_price', + name: 'brand_name', + }[sortBy] || 'product_count'; + + const result = await this.pool.query(` + SELECT + brand_id AS "brandId", + brand_name AS "brandName", + brand_slug AS "brandSlug", + store_count AS "storeCount", + product_count AS "productCount", + avg_price AS "avgPrice", + first_seen_in_state AS "firstSeenInState", + last_seen_in_state AS "lastSeenInState" + FROM v_brand_state_presence + WHERE state = $1 + ORDER BY ${sortColumn} ${sortDir === 'asc' ? 'ASC' : 'DESC'} + LIMIT $2 OFFSET $3 + `, [state, limit, offset]); + + return result.rows; + } + + /** + * Get brand penetration across all states + */ + async getBrandStatePenetration(brandId: number): Promise { + const result = await this.pool.query(` + SELECT + state, + state_name AS "stateName", + total_stores AS "totalStores", + stores_with_brand AS "storesWithBrand", + penetration_pct AS "penetrationPct", + product_count AS "productCount", + avg_price AS "avgPrice" + FROM fn_brand_state_penetration($1) + `, [brandId]); + + return result.rows; + } + + /** + * Compare a brand across multiple states + */ + async compareBrandAcrossStates( + brandId: number, + states: string[] + ): Promise { + // Get brand info + const brandResult = await this.pool.query(` + SELECT id, name FROM brands WHERE id = $1 + `, [brandId]); + + if (brandResult.rows.length === 0) { + throw new Error(`Brand ${brandId} not found`); + } + + const brand = brandResult.rows[0]; + + // Get penetration for specified states + const allPenetration = await this.getBrandStatePenetration(brandId); + const filteredStates = allPenetration.filter(p => states.includes(p.state)); + + // Calculate national metrics + const nationalResult = await this.pool.query(` + SELECT + COUNT(DISTINCT d.id) AS total_stores, + COUNT(DISTINCT CASE WHEN sp.brand_id = $1 THEN d.id END) AS stores_with_brand, + AVG(sp.price_rec) FILTER (WHERE sp.brand_id = $1) AS avg_price + FROM dispensaries d + LEFT JOIN store_products sp ON d.id = sp.dispensary_id + WHERE d.state IS NOT NULL + `, [brandId]); + + const nationalData = nationalResult.rows[0]; + const nationalPenetration = nationalData.total_stores > 0 + ? (nationalData.stores_with_brand / nationalData.total_stores) * 100 + : 0; + + // Find best/worst states + const sortedByPenetration = [...filteredStates].sort( + (a, b) => b.penetrationPct - a.penetrationPct + ); + + return { + brandId, + brandName: brand.name, + states: filteredStates, + nationalPenetration: Math.round(nationalPenetration * 100) / 100, + nationalAvgPrice: nationalData.avg_price + ? Math.round(nationalData.avg_price * 100) / 100 + : null, + bestPerformingState: sortedByPenetration[0]?.state || null, + worstPerformingState: sortedByPenetration[sortedByPenetration.length - 1]?.state || null, + }; + } + + // ========================================================================= + // Category Queries + // ========================================================================= + + /** + * Get categories in a specific state + */ + async getCategoriesByState(state: string, options: StateQueryOptions = {}): Promise { + const { limit = 50, offset = 0, sortBy = 'productCount', sortDir = 'desc' } = options; + + const sortColumn = { + productCount: 'product_count', + storeCount: 'store_count', + avgPrice: 'avg_price', + category: 'category', + }[sortBy] || 'product_count'; + + const result = await this.pool.query(` + SELECT + category, + product_count AS "productCount", + store_count AS "storeCount", + avg_price AS "avgPrice", + in_stock_count AS "inStockCount", + on_special_count AS "onSpecialCount" + FROM v_category_state_distribution + WHERE state = $1 + ORDER BY ${sortColumn} ${sortDir === 'asc' ? 'ASC' : 'DESC'} + LIMIT $2 OFFSET $3 + `, [state, limit, offset]); + + return result.rows; + } + + /** + * Compare a category across multiple states + */ + async compareCategoryAcrossStates( + category: string, + states: string[] + ): Promise { + const result = await this.pool.query(` + SELECT + v.state, + s.name AS "stateName", + v.category, + v.product_count AS "productCount", + v.store_count AS "storeCount", + v.avg_price AS "avgPrice", + ROUND(v.product_count::NUMERIC / SUM(v.product_count) OVER () * 100, 2) AS "marketShare" + FROM v_category_state_distribution v + JOIN states s ON v.state = s.code + WHERE v.category = $1 + AND v.state = ANY($2) + ORDER BY v.product_count DESC + `, [category, states]); + + // Get national totals + const nationalResult = await this.pool.query(` + SELECT + COUNT(DISTINCT sp.id) AS product_count, + AVG(sp.price_rec) AS avg_price + FROM store_products sp + WHERE sp.category_raw = $1 + `, [category]); + + const national = nationalResult.rows[0]; + + // Find dominant state + const dominantState = result.rows.length > 0 ? result.rows[0].state : null; + + return { + category, + states: result.rows, + nationalProductCount: parseInt(national.product_count || '0'), + nationalAvgPrice: national.avg_price + ? Math.round(national.avg_price * 100) / 100 + : null, + dominantState, + }; + } + + // ========================================================================= + // Store Queries + // ========================================================================= + + /** + * Get stores in a specific state + */ + async getStoresByState(state: string, options: StateQueryOptions = {}): Promise { + const { limit = 100, offset = 0, includeInactive = false, sortBy = 'productCount', sortDir = 'desc' } = options; + + const sortColumn = { + productCount: 'product_count', + brandCount: 'brand_count', + avgPrice: 'avg_price', + name: 'dispensary_name', + city: 'city', + lastCrawl: 'last_crawl_at', + }[sortBy] || 'product_count'; + + let whereClause = 'WHERE state = $1'; + if (!includeInactive) { + whereClause += ` AND crawl_status != 'disabled'`; + } + + const result = await this.pool.query(` + SELECT + dispensary_id AS "dispensaryId", + dispensary_name AS "dispensaryName", + dispensary_slug AS "dispensarySlug", + state, + city, + menu_type AS "menuType", + crawl_status AS "crawlStatus", + last_crawl_at AS "lastCrawlAt", + product_count AS "productCount", + in_stock_count AS "inStockCount", + brand_count AS "brandCount", + avg_price AS "avgPrice", + special_count AS "specialCount" + FROM v_store_state_summary + ${whereClause} + ORDER BY ${sortColumn} ${sortDir === 'asc' ? 'ASC' : 'DESC'} NULLS LAST + LIMIT $2 OFFSET $3 + `, [state, limit, offset]); + + return result.rows; + } + + // ========================================================================= + // Price Analytics + // ========================================================================= + + /** + * Get price distribution by state + */ + async getStorePriceDistribution( + state: string, + options: { category?: string; brandId?: number } = {} + ): Promise { + const { category, brandId } = options; + + const result = await this.pool.query(` + SELECT * FROM fn_national_price_comparison($1, $2) + WHERE state = $3 + `, [category || null, brandId || null, state]); + + return result.rows.map(row => ({ + state: row.state, + stateName: row.state_name, + productCount: parseInt(row.product_count), + avgPrice: parseFloat(row.avg_price), + minPrice: parseFloat(row.min_price), + maxPrice: parseFloat(row.max_price), + medianPrice: parseFloat(row.median_price), + priceStddev: parseFloat(row.price_stddev), + })); + } + + /** + * Get national price comparison across all states + */ + async getNationalPriceComparison( + options: { category?: string; brandId?: number } = {} + ): Promise { + const { category, brandId } = options; + + const result = await this.pool.query(` + SELECT * FROM fn_national_price_comparison($1, $2) + `, [category || null, brandId || null]); + + return result.rows.map(row => ({ + state: row.state, + stateName: row.state_name, + productCount: parseInt(row.product_count), + avgPrice: parseFloat(row.avg_price), + minPrice: parseFloat(row.min_price), + maxPrice: parseFloat(row.max_price), + medianPrice: parseFloat(row.median_price), + priceStddev: parseFloat(row.price_stddev), + })); + } + + // ========================================================================= + // National Analytics + // ========================================================================= + + /** + * Get national summary across all states + */ + async getNationalSummary(): Promise { + const stateMetrics = await this.getAllStateMetrics(); + + const result = await this.pool.query(` + SELECT + COUNT(DISTINCT s.code) AS total_states, + COUNT(DISTINCT CASE WHEN EXISTS ( + SELECT 1 FROM dispensaries d WHERE d.state = s.code AND d.menu_type IS NOT NULL + ) THEN s.code END) AS active_states, + (SELECT COUNT(*) FROM dispensaries WHERE state IS NOT NULL) AS total_stores, + (SELECT COUNT(*) FROM store_products sp + JOIN dispensaries d ON sp.dispensary_id = d.id + WHERE d.state IS NOT NULL) AS total_products, + (SELECT COUNT(DISTINCT brand_id) FROM store_products sp + JOIN dispensaries d ON sp.dispensary_id = d.id + WHERE d.state IS NOT NULL AND sp.brand_id IS NOT NULL) AS total_brands, + (SELECT AVG(price_rec) FROM store_products sp + JOIN dispensaries d ON sp.dispensary_id = d.id + WHERE d.state IS NOT NULL AND sp.price_rec > 0) AS avg_price_national + FROM states s + `); + + const data = result.rows[0]; + + return { + totalStates: parseInt(data.total_states), + activeStates: parseInt(data.active_states), + totalStores: parseInt(data.total_stores), + totalProducts: parseInt(data.total_products), + totalBrands: parseInt(data.total_brands), + avgPriceNational: data.avg_price_national + ? Math.round(parseFloat(data.avg_price_national) * 100) / 100 + : null, + stateMetrics, + }; + } + + /** + * Get heatmap data for a specific metric + */ + async getStateHeatmapData( + metric: 'stores' | 'products' | 'brands' | 'avgPrice' | 'penetration', + options: { brandId?: number; category?: string } = {} + ): Promise { + let query: string; + let params: any[] = []; + + switch (metric) { + case 'stores': + query = ` + SELECT state, state_name AS "stateName", dispensary_count AS value, 'stores' AS label + FROM mv_state_metrics + WHERE state IS NOT NULL + ORDER BY state + `; + break; + + case 'products': + query = ` + SELECT state, state_name AS "stateName", total_products AS value, 'products' AS label + FROM mv_state_metrics + WHERE state IS NOT NULL + ORDER BY state + `; + break; + + case 'brands': + query = ` + SELECT state, state_name AS "stateName", unique_brands AS value, 'brands' AS label + FROM mv_state_metrics + WHERE state IS NOT NULL + ORDER BY state + `; + break; + + case 'avgPrice': + query = ` + SELECT state, state_name AS "stateName", avg_price_rec AS value, 'avg price' AS label + FROM mv_state_metrics + WHERE state IS NOT NULL AND avg_price_rec IS NOT NULL + ORDER BY state + `; + break; + + case 'penetration': + if (!options.brandId) { + throw new Error('brandId required for penetration heatmap'); + } + query = ` + SELECT state, state_name AS "stateName", penetration_pct AS value, 'penetration %' AS label + FROM fn_brand_state_penetration($1) + ORDER BY state + `; + params = [options.brandId]; + break; + + default: + throw new Error(`Unknown metric: ${metric}`); + } + + const result = await this.pool.query(query, params); + return result.rows; + } + + /** + * Get national penetration trend for a brand + */ + async getNationalPenetrationTrend( + brandId: number, + options: { days?: number } = {} + ): Promise { + const { days = 30 } = options; + + // Get brand info + const brandResult = await this.pool.query(` + SELECT id, name FROM brands WHERE id = $1 + `, [brandId]); + + if (brandResult.rows.length === 0) { + throw new Error(`Brand ${brandId} not found`); + } + + // Get historical data from snapshots + const result = await this.pool.query(` + WITH daily_presence AS ( + SELECT + DATE(sps.captured_at) AS date, + COUNT(DISTINCT d.state) AS states_present, + COUNT(DISTINCT d.id) AS stores_with_brand + FROM store_product_snapshots sps + JOIN dispensaries d ON sps.dispensary_id = d.id + JOIN store_products sp ON sps.store_product_id = sp.id + WHERE sp.brand_id = $1 + AND sps.captured_at > NOW() - INTERVAL '1 day' * $2 + AND d.state IS NOT NULL + GROUP BY DATE(sps.captured_at) + ), + daily_totals AS ( + SELECT + DATE(sps.captured_at) AS date, + COUNT(DISTINCT d.id) AS total_stores + FROM store_product_snapshots sps + JOIN dispensaries d ON sps.dispensary_id = d.id + WHERE sps.captured_at > NOW() - INTERVAL '1 day' * $2 + AND d.state IS NOT NULL + GROUP BY DATE(sps.captured_at) + ) + SELECT + dp.date, + dp.states_present, + dt.total_stores, + ROUND(dp.stores_with_brand::NUMERIC / NULLIF(dt.total_stores, 0) * 100, 2) AS penetration_pct + FROM daily_presence dp + JOIN daily_totals dt ON dp.date = dt.date + ORDER BY dp.date + `, [brandId, days]); + + return { + brandId, + brandName: brandResult.rows[0].name, + dataPoints: result.rows.map(row => ({ + date: row.date.toISOString().split('T')[0], + statesPresent: parseInt(row.states_present), + totalStores: parseInt(row.total_stores), + penetrationPct: parseFloat(row.penetration_pct || '0'), + })), + }; + } + + // ========================================================================= + // Utility Methods + // ========================================================================= + + /** + * Refresh materialized views + * Uses direct REFRESH MATERIALIZED VIEW for compatibility + */ + async refreshMetrics(): Promise { + // Use direct refresh command instead of function call for better compatibility + // CONCURRENTLY requires a unique index (idx_mv_state_metrics_state exists) + await this.pool.query('REFRESH MATERIALIZED VIEW CONCURRENTLY mv_state_metrics'); + } + + /** + * Validate state code + */ + async isValidState(state: string): Promise { + const result = await this.pool.query(` + SELECT 1 FROM states WHERE code = $1 + `, [state]); + return result.rows.length > 0; + } +} diff --git a/backend/src/multi-state/types.ts b/backend/src/multi-state/types.ts new file mode 100644 index 00000000..c47b6004 --- /dev/null +++ b/backend/src/multi-state/types.ts @@ -0,0 +1,199 @@ +/** + * Multi-State Module Types + * Phase 4: Multi-State Expansion + */ + +// Core state types +export interface State { + code: string; + name: string; +} + +export interface StateMetrics { + state: string; + stateName: string; + storeCount: number; + dutchieStores: number; + activeStores: number; + totalProducts: number; + inStockProducts: number; + onSpecialProducts: number; + uniqueBrands: number; + uniqueCategories: number; + avgPriceRec: number | null; + minPriceRec: number | null; + maxPriceRec: number | null; + refreshedAt: Date; +} + +export interface StateSummary extends StateMetrics { + recentCrawls: number; + failedCrawls: number; + lastCrawlAt: Date | null; + topBrands: BrandInState[]; + topCategories: CategoryInState[]; +} + +// Brand analytics +export interface BrandInState { + brandId: number; + brandName: string; + brandSlug: string; + storeCount: number; + productCount: number; + avgPrice: number | null; + firstSeenInState: Date | null; + lastSeenInState: Date | null; +} + +export interface BrandStatePenetration { + state: string; + stateName: string; + totalStores: number; + storesWithBrand: number; + penetrationPct: number; + productCount: number; + avgPrice: number | null; +} + +export interface BrandCrossStateComparison { + brandId: number; + brandName: string; + states: BrandStatePenetration[]; + nationalPenetration: number; + nationalAvgPrice: number | null; + bestPerformingState: string | null; + worstPerformingState: string | null; +} + +// Category analytics +export interface CategoryInState { + category: string; + productCount: number; + storeCount: number; + avgPrice: number | null; + inStockCount: number; + onSpecialCount: number; +} + +export interface CategoryStateDist { + state: string; + stateName: string; + category: string; + productCount: number; + storeCount: number; + avgPrice: number | null; + marketShare: number; +} + +export interface CategoryCrossStateComparison { + category: string; + states: CategoryStateDist[]; + nationalProductCount: number; + nationalAvgPrice: number | null; + dominantState: string | null; +} + +// Store analytics +export interface StoreInState { + dispensaryId: number; + dispensaryName: string; + dispensarySlug: string; + state: string; + city: string; + menuType: string | null; + crawlStatus: string; + lastCrawlAt: Date | null; + productCount: number; + inStockCount: number; + brandCount: number; + avgPrice: number | null; + specialCount: number; +} + +// Price analytics +export interface StatePriceDistribution { + state: string; + stateName: string; + productCount: number; + avgPrice: number; + minPrice: number; + maxPrice: number; + medianPrice: number; + priceStddev: number; +} + +export interface NationalPriceTrend { + date: string; + states: { + [stateCode: string]: { + avgPrice: number; + productCount: number; + }; + }; +} + +export interface ProductPriceByState { + productId: string; + productName: string; + states: { + state: string; + stateName: string; + avgPrice: number; + minPrice: number; + maxPrice: number; + storeCount: number; + }[]; +} + +// National metrics +export interface NationalSummary { + totalStates: number; + activeStates: number; + totalStores: number; + totalProducts: number; + totalBrands: number; + avgPriceNational: number | null; + stateMetrics: StateMetrics[]; +} + +export interface NationalPenetrationTrend { + brandId: number; + brandName: string; + dataPoints: { + date: string; + statesPresent: number; + totalStores: number; + penetrationPct: number; + }[]; +} + +// Heatmap data +export interface StateHeatmapData { + state: string; + stateName: string; + value: number; + label: string; +} + +// Query options +export interface StateQueryOptions { + state?: string; + states?: string[]; + includeInactive?: boolean; + limit?: number; + offset?: number; + sortBy?: string; + sortDir?: 'asc' | 'desc'; + dateFrom?: Date; + dateTo?: Date; +} + +export interface CrossStateQueryOptions { + states: string[]; + metric: 'price' | 'penetration' | 'products' | 'stores'; + category?: string; + brandId?: number; + dateFrom?: Date; + dateTo?: Date; +} diff --git a/cannaiq/src/App.tsx b/cannaiq/src/App.tsx index da0a188b..d584bd72 100755 --- a/cannaiq/src/App.tsx +++ b/cannaiq/src/App.tsx @@ -26,6 +26,21 @@ import { DutchieAZStores } from './pages/DutchieAZStores'; import { DutchieAZStoreDetail } from './pages/DutchieAZStoreDetail'; import { WholesaleAnalytics } from './pages/WholesaleAnalytics'; import { Users } from './pages/Users'; +import { OrchestratorDashboard } from './pages/OrchestratorDashboard'; +import { OrchestratorProducts } from './pages/OrchestratorProducts'; +import { OrchestratorBrands } from './pages/OrchestratorBrands'; +import { OrchestratorStores } from './pages/OrchestratorStores'; +import { ChainsDashboard } from './pages/ChainsDashboard'; +import { IntelligenceBrands } from './pages/IntelligenceBrands'; +import { IntelligencePricing } from './pages/IntelligencePricing'; +import { IntelligenceStores } from './pages/IntelligenceStores'; +import { SyncInfoPanel } from './pages/SyncInfoPanel'; +import NationalDashboard from './pages/NationalDashboard'; +import StateHeatmap from './pages/StateHeatmap'; +import CrossStateCompare from './pages/CrossStateCompare'; +import { Discovery } from './pages/Discovery'; +import { WorkersDashboard } from './pages/WorkersDashboard'; +import { ScraperOverviewDashboard } from './pages/ScraperOverviewDashboard'; import { PrivateRoute } from './components/PrivateRoute'; export default function App() { @@ -59,6 +74,29 @@ export default function App() { } /> } /> } /> + {/* National / Multi-State routes */} + } /> + } /> + } /> + {/* Admin routes */} + } /> + } /> + } /> + } /> + } /> + } /> + {/* Intelligence routes */} + } /> + } /> + } /> + } /> + } /> + {/* Discovery routes */} + } /> + {/* Workers Dashboard */} + } /> + {/* Scraper Overview Dashboard (new primary) */} + } /> } /> diff --git a/cannaiq/src/components/Layout.tsx b/cannaiq/src/components/Layout.tsx index f54037bf..8dd97a31 100755 --- a/cannaiq/src/components/Layout.tsx +++ b/cannaiq/src/components/Layout.tsx @@ -2,6 +2,7 @@ import { ReactNode, useEffect, useState } from 'react'; import { useNavigate, useLocation } from 'react-router-dom'; import { useAuthStore } from '../store/authStore'; import { api } from '../lib/api'; +import { StateSelector } from './StateSelector'; import { LayoutDashboard, Store, @@ -20,7 +21,13 @@ import { LogOut, CheckCircle, Key, - Users + Users, + Globe, + Map, + Search, + HardHat, + Gauge, + Archive } from 'lucide-react'; interface LayoutProps { @@ -132,6 +139,11 @@ export function Layout({ children }: LayoutProps) {

{user?.email}

+ {/* State Selector */} +
+ +
+ {/* Navigation */} {/* Logout */} diff --git a/cannaiq/src/components/StoreOrchestratorPanel.tsx b/cannaiq/src/components/StoreOrchestratorPanel.tsx new file mode 100644 index 00000000..2768a7f3 --- /dev/null +++ b/cannaiq/src/components/StoreOrchestratorPanel.tsx @@ -0,0 +1,1115 @@ +import { useState, useEffect, useMemo } from 'react'; +import { api } from '../lib/api'; +import { + X, + FileText, + Settings, + Code, + CheckCircle, + XCircle, + Clock, + AlertTriangle, + ChevronDown, + ChevronRight, + Loader2, + Copy, + ExternalLink, + Workflow, + Play, + FlaskConical, + Rocket, + ToggleLeft, + ToggleRight, + Zap, + Database, + Bug, + FileJson, +} from 'lucide-react'; +import { WorkflowStepper, analyzeTracePhases, getPhasesForCrawlerType } from './WorkflowStepper'; + +interface StoreInfo { + id: number; + name: string; + city: string; + state: string; + provider: string; + provider_raw?: string | null; + provider_display?: string; + platformDispensaryId: string | null; + status: string; + profileId: number | null; + profileKey: string | null; + sandboxAttempts?: number; + nextRetryAt?: string | null; + lastCrawlAt: string | null; + lastSuccessAt: string | null; + lastFailureAt: string | null; + failedAt?: string | null; + consecutiveFailures?: number; + productCount: number; +} + +interface TraceStep { + step: number; + action: string; + description: string; + timestamp: number; + duration_ms?: number; + input: Record; + output: Record | null; + what: string; + why: string; + where: string; + how: string; + when: string; + status: string; + error?: string; +} + +interface TraceSummary { + id: number; + dispensaryId: number; + runId: string; + profileId: number | null; + profileKey: string | null; + crawlerModule: string | null; + stateAtStart: string; + stateAtEnd: string; + totalSteps: number; + durationMs: number; + success: boolean; + errorMessage: string | null; + productsFound: number; + startedAt: string; + completedAt: string | null; + trace: TraceStep[]; +} + +interface ProfileData { + dispensaryId: number; + dispensaryName: string; + hasProfile: boolean; + activeProfileId: number | null; + menuType?: string; + platformDispensaryId?: string; + profile?: { + id: number; + profileKey: string; + profileName: string; + platform: string; + version: number; + status: string; + config: Record; + enabled: boolean; + sandboxAttemptCount: number; + nextRetryAt: string | null; + createdAt: string; + updatedAt: string; + }; +} + +interface CrawlerModuleData { + hasModule: boolean; + profileKey?: string; + platform?: string; + fileName?: string; + filePath?: string; + content?: string; + lines?: number; + error?: string; + expectedPath?: string; +} + +interface SnapshotData { + id: number; + productId: number; + productName: string; + brandName: string | null; + crawledAt: string; + stockStatus: string; + regularPrice: number | null; + salePrice: number | null; + rawPayload: Record | null; +} + +interface StoreOrchestratorPanelProps { + store: StoreInfo; + activeTab: 'control' | 'trace' | 'profile' | 'module' | 'debug'; + onTabChange: (tab: 'control' | 'trace' | 'profile' | 'module' | 'debug') => void; + onClose: () => void; +} + +export function StoreOrchestratorPanel({ + store, + activeTab, + onTabChange, + onClose, +}: StoreOrchestratorPanelProps) { + const [trace, setTrace] = useState(null); + const [profile, setProfile] = useState(null); + const [crawlerModule, setCrawlerModule] = useState(null); + const [loading, setLoading] = useState(false); + const [error, setError] = useState(null); + const [expandedSteps, setExpandedSteps] = useState>(new Set()); + const [selectedPhase, setSelectedPhase] = useState(null); + + // Crawler control state + const [crawlRunning, setCrawlRunning] = useState(false); + const [crawlResult, setCrawlResult] = useState<{ success: boolean; message: string } | null>(null); + const [showAutopromoteConfirm, setShowAutopromoteConfirm] = useState(false); + const [allowAutopromote, setAllowAutopromote] = useState(false); + + // Debug tab state + const [snapshots, setSnapshots] = useState([]); + const [selectedSnapshot, setSelectedSnapshot] = useState(null); + const [copiedPayload, setCopiedPayload] = useState(false); + + // Filter trace steps by selected phase + const filteredTraceSteps = useMemo(() => { + if (!trace?.trace || !selectedPhase) return trace?.trace || []; + + const phases = getPhasesForCrawlerType(trace.crawlerModule); + const phase = phases.find(p => p.key === selectedPhase); + if (!phase) return trace.trace; + + return trace.trace.filter(step => + phase.actions.some(action => + step.action.toLowerCase().includes(action.toLowerCase()) + ) + ); + }, [trace, selectedPhase]); + + useEffect(() => { + loadTabData(); + }, [store.id, activeTab]); + + const loadTabData = async () => { + setLoading(true); + setError(null); + + try { + switch (activeTab) { + case 'trace': + const traceData = await api.getOrchestratorDispensaryTraceLatest(store.id); + setTrace(traceData); + // Auto-expand failed steps + if (traceData?.trace) { + const failedSteps = traceData.trace + .filter((s) => s.status === 'failed') + .map((s) => s.step); + setExpandedSteps(new Set(failedSteps)); + } + break; + case 'profile': + const profileData = await api.getOrchestratorDispensaryProfile(store.id); + setProfile(profileData); + break; + case 'module': + const moduleData = await api.getOrchestratorCrawlerModule(store.id); + setCrawlerModule(moduleData); + break; + case 'debug': + const snapshotsData = await api.getStoreSnapshots(store.id, { limit: 50 }); + setSnapshots(snapshotsData.snapshots || []); + setSelectedSnapshot(null); + break; + } + } catch (err: any) { + setError(err.message || `Failed to load ${activeTab} data`); + } finally { + setLoading(false); + } + }; + + const toggleStep = (stepNum: number) => { + setExpandedSteps((prev) => { + const next = new Set(prev); + if (next.has(stepNum)) { + next.delete(stepNum); + } else { + next.add(stepNum); + } + return next; + }); + }; + + const getStatusIcon = (status: string) => { + switch (status) { + case 'completed': + return ; + case 'failed': + return ; + case 'skipped': + return ; + case 'running': + return ; + default: + return ; + } + }; + + const getStatusBadge = (status: string) => { + switch (status) { + case 'completed': + return 'bg-green-100 text-green-800'; + case 'failed': + return 'bg-red-100 text-red-800'; + case 'skipped': + return 'bg-yellow-100 text-yellow-800'; + case 'running': + return 'bg-blue-100 text-blue-800'; + default: + return 'bg-gray-100 text-gray-800'; + } + }; + + const formatDuration = (ms?: number) => { + if (!ms) return '-'; + if (ms < 1000) return `${ms}ms`; + return `${(ms / 1000).toFixed(2)}s`; + }; + + const formatTimestamp = (ts: string) => { + return new Date(ts).toLocaleString(); + }; + + const copyToClipboard = (text: string) => { + navigator.clipboard.writeText(text); + }; + + // Crawler control handlers + const runCrawl = async (mode: 'sandbox' | 'production') => { + setCrawlRunning(true); + setCrawlResult(null); + + try { + // Use existing crawl endpoint with mode parameter + const result = await api.triggerOrchestratorCrawl(store.id, { mode }); + setCrawlResult({ + success: true, + message: `${mode === 'sandbox' ? 'Sandbox' : 'Production'} crawl initiated successfully`, + }); + + // Refresh trace data after a short delay + setTimeout(() => { + if (activeTab === 'trace') { + loadTabData(); + } + }, 2000); + } catch (err: any) { + setCrawlResult({ + success: false, + message: err.message || `Failed to start ${mode} crawl`, + }); + } finally { + setCrawlRunning(false); + } + }; + + const handleAutopromoteToggle = async () => { + if (!allowAutopromote) { + // Show confirmation dialog + setShowAutopromoteConfirm(true); + } else { + // Disable autopromote + try { + await api.updateOrchestratorAutopromote(store.id, false); + setAllowAutopromote(false); + } catch (err: any) { + console.error('Failed to update autopromote:', err); + } + } + }; + + const confirmAutopromote = async () => { + try { + await api.updateOrchestratorAutopromote(store.id, true); + setAllowAutopromote(true); + } catch (err: any) { + console.error('Failed to enable autopromote:', err); + } finally { + setShowAutopromoteConfirm(false); + } + }; + + const handlePhaseClick = (phaseKey: string, firstStepIndex?: number) => { + // Toggle phase filter + if (selectedPhase === phaseKey) { + setSelectedPhase(null); + } else { + setSelectedPhase(phaseKey); + // Auto-expand the first step of this phase + if (firstStepIndex !== undefined) { + setExpandedSteps(new Set([firstStepIndex])); + } + } + }; + + const renderControlTab = () => { + const getStatusColor = (status: string) => { + switch (status) { + case 'production': return 'text-green-600 bg-green-100'; + case 'sandbox': return 'text-yellow-600 bg-yellow-100'; + case 'needs_manual': return 'text-orange-600 bg-orange-100'; + case 'disabled': return 'text-gray-600 bg-gray-100'; + case 'legacy': return 'text-blue-600 bg-blue-100'; + default: return 'text-gray-600 bg-gray-100'; + } + }; + + return ( +
+ {/* Crawler Status */} +
+

+ + Crawler Status +

+
+
+

Status

+ + {store.status?.toUpperCase() || 'UNKNOWN'} + +
+
+

Profile Key

+

{store.profileKey || '-'}

+
+
+

Provider

+

{store.provider_display || store.provider || '-'}

+
+
+

Sandbox Attempts

+

{store.sandboxAttempts || 0}

+
+
+

Last Success

+

+ {store.lastSuccessAt ? formatTimestamp(store.lastSuccessAt) : 'Never'} +

+
+
+

Last Failure

+

+ {store.lastFailureAt ? formatTimestamp(store.lastFailureAt) : 'None'} +

+
+
+

Consecutive Failures

+

0 ? 'text-red-600' : 'text-green-600'}`}> + {store.consecutiveFailures || 0} +

+
+
+

Products

+

{store.productCount?.toLocaleString() || 0}

+
+
+
+ + {/* Crawler Actions */} +
+

+ + Run Crawler +

+
+ + +
+ {store.status !== 'production' && ( +

+ Production crawl only available when store is in production status. +

+ )} + + {/* Crawl Result */} + {crawlResult && ( +
+ {crawlResult.success ? ( + + ) : ( + + )} + {crawlResult.message} +
+ )} +
+ + {/* Autopromote Toggle */} +
+
+
+

+ Auto-Promote + {allowAutopromote ? ( + + ) : ( + + )} +

+

+ Automatically promote from sandbox to production when validation passes +

+
+ +
+
+ + {/* Confirmation Dialog */} + {showAutopromoteConfirm && ( +
+
+

Enable Auto-Promote?

+

+ Enabling auto-promote will automatically move this store from sandbox to production + when sandbox validation passes. Make sure the sandbox crawl is consistently passing + before enabling this. +

+
+ + +
+
+
+ )} + + {/* Quick Stats */} +
+

+ + Quick Info +

+
+

Store ID: {store.id}

+

Platform ID: {store.platformDispensaryId || 'Not set'}

+

Profile ID: {store.profileId || 'Not set'}

+ {store.nextRetryAt && ( +

Next Retry: {formatTimestamp(store.nextRetryAt)}

+ )} +
+
+
+ ); + }; + + const renderTraceTab = () => { + if (!trace) { + return ( +
+ +

No trace found for this store

+

Run a crawl first to generate a trace

+
+ ); + } + + return ( +
+ {/* Workflow Stepper */} +
+
+ +

Workflow

+
+ +
+ + {/* Trace Summary */} +
+
+
+

Status

+

+ {trace.success ? 'Success' : 'Failed'} +

+
+
+

Duration

+

{formatDuration(trace.durationMs)}

+
+
+

Products

+

{trace.productsFound}

+
+
+

Steps

+

{trace.totalSteps}

+
+
+

Profile Key

+

{trace.profileKey || '-'}

+
+
+

State Change

+

{trace.stateAtStart} → {trace.stateAtEnd}

+
+
+ {trace.errorMessage && ( +
+ {trace.errorMessage} +
+ )} +
+ + {/* Steps */} +
+
+

+ Steps ({filteredTraceSteps.length}{selectedPhase ? ` of ${trace.trace.length}` : ''}) +

+ {selectedPhase && ( + + )} +
+
+ {filteredTraceSteps.map((step) => ( +
+ + + {expandedSteps.has(step.step) && ( +
+
+
+

WHAT

+

{step.what}

+
+
+

WHY

+

{step.why}

+
+
+

WHERE

+

{step.where}

+
+
+

HOW

+

{step.how}

+
+
+ + {step.error && ( +
+ Error: {step.error} +
+ )} + + {Object.keys(step.input || {}).length > 0 && ( +
+

INPUT

+ 
+                          {JSON.stringify(step.input, null, 2)}
+
+
+ )} + + {step.output && Object.keys(step.output).length > 0 && ( +
+

OUTPUT

+ 
+                          {JSON.stringify(step.output, null, 2)}
+
+
+ )} +
+ )} +
+ ))} +
+
+
+ ); + }; + + const renderProfileTab = () => { + if (!profile) { + return ( +
+ +

No profile data available

+
+ ); + } + + return ( +
+ {/* Basic Info */} +
+

Dispensary Info

+
+
+

ID

+

{profile.dispensaryId}

+
+
+

Has Profile

+

+ {profile.hasProfile ? 'Yes' : 'No'} +

+
+
+

Menu Type

+

{profile.menuType || '-'}

+
+
+

Platform ID

+

+ {profile.platformDispensaryId || '-'} +

+
+
+
+ + {/* Profile Details */} + {profile.profile && ( +
+

Profile Config

+
+
+

Profile Key

+

{profile.profile.profileKey}

+
+
+

Platform

+

{profile.profile.platform}

+
+
+

Status

+ + {profile.profile.status} + +
+
+

Version

+

{profile.profile.version}

+
+
+

Sandbox Attempts

+

{profile.profile.sandboxAttemptCount}

+
+
+

Enabled

+

+ {profile.profile.enabled ? 'Yes' : 'No'} +

+
+
+ + {/* Config JSON */} + {profile.profile.config && Object.keys(profile.profile.config).length > 0 && ( +
+

Config

+ 
+                  {JSON.stringify(profile.profile.config, null, 2)}
+
+
+ )} +
+ )} +
+ ); + }; + + const renderModuleTab = () => { + if (!crawlerModule) { + return ( +
+ +

No module data available

+
+ ); + } + + if (!crawlerModule.hasModule) { + return ( +
+ +

{crawlerModule.error || 'No per-store crawler module found'}

+ {crawlerModule.expectedPath && ( +

+ Expected: {crawlerModule.expectedPath} +

+ )} +
+ ); + } + + return ( +
+ {/* Module Info */} +
+
+
+

File

+

{crawlerModule.fileName}

+
+
+ +
+
+
+
+

Platform

+

{crawlerModule.platform}

+
+
+

Lines

+

{crawlerModule.lines}

+
+
+
+ + {/* Code Preview */} +
+

Source Code

+
+ 
+              {crawlerModule.content}
+
+
+
+
+ ); + }; + + const copyPayloadToClipboard = (payload: Record) => { + navigator.clipboard.writeText(JSON.stringify(payload, null, 2)); + setCopiedPayload(true); + setTimeout(() => setCopiedPayload(false), 2000); + }; + + const renderDebugTab = () => { + return ( +
+ {/* Snapshots Header */} +
+

+ + Recent Snapshots with Raw Payloads +

+ {snapshots.length} snapshots +
+ + {snapshots.length === 0 ? ( +
+ +

No snapshots available

+

Run a crawl to generate snapshots with raw payloads

+
+ ) : ( +
+ {/* Snapshot List */} +
+ + + + + + + + + + + + {snapshots.map((snapshot) => ( + setSelectedSnapshot(snapshot)} + > + + + + + + + ))} + +
ProductBrandStatusCrawled
+ {snapshot.productName} + + {snapshot.brandName || '-'} + + + {snapshot.stockStatus} + + + {new Date(snapshot.crawledAt).toLocaleString()} + + {snapshot.rawPayload ? ( + + ) : ( + - + )} +
+
+ + {/* Selected Snapshot Raw Payload */} + {selectedSnapshot && ( +
+
+
+ Snapshot #{selectedSnapshot.id} + {selectedSnapshot.productName} +
+ {selectedSnapshot.rawPayload && ( + + )} +
+
+ {selectedSnapshot.rawPayload ? ( +
+ 
+                        {JSON.stringify(selectedSnapshot.rawPayload, null, 2)}
+
+
+ ) : ( +
+ No raw payload stored for this snapshot +
+ )} +
+ + {/* Snapshot Metadata */} +
+
+

Product ID: {selectedSnapshot.productId}

+

Snapshot ID: {selectedSnapshot.id}

+

Price: ${selectedSnapshot.regularPrice?.toFixed(2) || '-'}

+ {selectedSnapshot.salePrice && ( +

Sale: ${selectedSnapshot.salePrice.toFixed(2)}

+ )} +
+
+
+ )} + + {!selectedSnapshot && snapshots.length > 0 && ( +
+ Click a snapshot above to view its raw payload +
+ )} +
+ )} +
+ ); + }; + + return ( +
+ {/* Header */} +
+
+

{store.name}

+

{store.city}, {store.state}

+
+ +
+ + {/* Tabs */} +
+ + + + + +
+ + {/* Content */} +
+ {loading ? ( +
+ + Loading... +
+ ) : error ? ( +
+ +

{error}

+ +
+ ) : ( + <> + {activeTab === 'control' && renderControlTab()} + {activeTab === 'trace' && renderTraceTab()} + {activeTab === 'profile' && renderProfileTab()} + {activeTab === 'module' && renderModuleTab()} + {activeTab === 'debug' && renderDebugTab()} + + )} +
+
+ ); +} diff --git a/cannaiq/src/components/WorkerRoleBadge.tsx b/cannaiq/src/components/WorkerRoleBadge.tsx new file mode 100644 index 00000000..237db9dc --- /dev/null +++ b/cannaiq/src/components/WorkerRoleBadge.tsx @@ -0,0 +1,138 @@ +/** + * WorkerRoleBadge Component + * + * Displays a badge for worker roles with consistent styling. + * + * Role mapping: + * product_sync / visibility_audit → [Products] + * store_discovery → [Discovery] + * entry_point_finder → [End Points] + * analytics_refresh → [Analytics] + * Unknown → [Other] + */ + +import React from 'react'; + +interface WorkerRoleBadgeProps { + role: string | null | undefined; + size?: 'sm' | 'md'; +} + +interface RoleConfig { + label: string; + bg: string; + color: string; +} + +const roleMapping: Record = { + product_sync: { + label: 'Products', + bg: '#dbeafe', + color: '#1e40af', + }, + visibility_audit: { + label: 'Products', + bg: '#dbeafe', + color: '#1e40af', + }, + store_discovery: { + label: 'Discovery', + bg: '#d1fae5', + color: '#065f46', + }, + entry_point_finder: { + label: 'End Points', + bg: '#fef3c7', + color: '#92400e', + }, + analytics_refresh: { + label: 'Analytics', + bg: '#ede9fe', + color: '#5b21b6', + }, +}; + +const defaultConfig: RoleConfig = { + label: 'Other', + bg: '#f3f4f6', + color: '#374151', +}; + +export function getRoleConfig(role: string | null | undefined): RoleConfig { + if (!role) return defaultConfig; + return roleMapping[role] || defaultConfig; +} + +export function WorkerRoleBadge({ role, size = 'sm' }: WorkerRoleBadgeProps) { + const config = getRoleConfig(role); + + const fontSize = size === 'sm' ? '11px' : '12px'; + const padding = size === 'sm' ? '2px 8px' : '4px 10px'; + + return ( + + {config.label} + + ); +} + +/** + * Format scope metadata for display + * + * @param metadata - Job metadata containing scope info + * @returns Human-readable scope string + */ +export function formatScope(metadata: any): string { + if (!metadata) return '-'; + + // Check for states scope + if (metadata.states && Array.isArray(metadata.states)) { + if (metadata.states.length <= 5) { + return metadata.states.join(', '); + } + return `${metadata.states.slice(0, 3).join(', ')} +${metadata.states.length - 3} more`; + } + + // Check for storeIds scope + if (metadata.storeIds && Array.isArray(metadata.storeIds)) { + return `${metadata.storeIds.length} stores (custom scope)`; + } + + // Check for dispensaryIds scope + if (metadata.dispensaryIds && Array.isArray(metadata.dispensaryIds)) { + return `${metadata.dispensaryIds.length} stores (custom scope)`; + } + + // Check for state field + if (metadata.state) { + return metadata.state; + } + + // Check for description + if (metadata.description) { + return metadata.description; + } + + // Check for scope.states + if (metadata.scope?.states && Array.isArray(metadata.scope.states)) { + if (metadata.scope.states.length <= 5) { + return metadata.scope.states.join(', '); + } + return `${metadata.scope.states.slice(0, 3).join(', ')} +${metadata.scope.states.length - 3} more`; + } + + return '-'; +} + +export default WorkerRoleBadge; diff --git a/cannaiq/src/lib/api.ts b/cannaiq/src/lib/api.ts index 1e1fbfae..3e5a357e 100755 --- a/cannaiq/src/lib/api.ts +++ b/cannaiq/src/lib/api.ts @@ -34,6 +34,20 @@ class ApiClient { return response.json(); } + // Generic HTTP methods (axios-style interface) + async get(endpoint: string): Promise<{ data: T }> { + const data = await this.request(endpoint); + return { data }; + } + + async post(endpoint: string, body?: any): Promise<{ data: T }> { + const data = await this.request(endpoint, { + method: 'POST', + body: body ? JSON.stringify(body) : undefined, + }); + return { data }; + } + // Auth async login(email: string, password: string) { return this.request<{ token: string; user: any }>('/api/auth/login', { @@ -671,6 +685,8 @@ class ApiClient { lastDurationMs: number | null; nextRunAt: string | null; jobConfig: Record | null; + workerName: string | null; + workerRole: string | null; createdAt: string; updatedAt: string; }>; @@ -1065,6 +1081,1008 @@ class ApiClient { method: 'DELETE', }); } + + // Orchestrator Traces + async getDispensaryTraceLatest(dispensaryId: number) { + return this.request<{ + id: number; + dispensaryId: number; + runId: string; + profileId: number | null; + profileKey: string | null; + crawlerModule: string | null; + stateAtStart: string; + stateAtEnd: string; + totalSteps: number; + durationMs: number; + success: boolean; + errorMessage: string | null; + productsFound: number; + startedAt: string; + completedAt: string | null; + trace: Array<{ + step: number; + action: string; + description: string; + timestamp: number; + duration_ms?: number; + input: Record; + output: Record | null; + what: string; + why: string; + where: string; + how: string; + when: string; + status: string; + error?: string; + }>; + }>(`/api/az/admin/dispensaries/${dispensaryId}/crawl-trace/latest`); + } + + async getDispensaryTraces(dispensaryId: number, params?: { limit?: number; offset?: number }) { + const searchParams = new URLSearchParams(); + if (params?.limit) searchParams.append('limit', params.limit.toString()); + if (params?.offset) searchParams.append('offset', params.offset.toString()); + const queryString = searchParams.toString() ? `?${searchParams.toString()}` : ''; + return this.request<{ + traces: Array<{ + id: number; + dispensaryId: number; + runId: string; + profileKey: string | null; + stateAtStart: string; + stateAtEnd: string; + totalSteps: number; + durationMs: number; + success: boolean; + errorMessage: string | null; + productsFound: number; + startedAt: string; + completedAt: string | null; + }>; + total: number; + }>(`/api/az/admin/dispensaries/${dispensaryId}/crawl-traces${queryString}`); + } + + async getTraceById(traceId: number) { + return this.request<{ + id: number; + dispensaryId: number; + runId: string; + profileId: number | null; + profileKey: string | null; + crawlerModule: string | null; + stateAtStart: string; + stateAtEnd: string; + totalSteps: number; + durationMs: number; + success: boolean; + errorMessage: string | null; + productsFound: number; + startedAt: string; + completedAt: string | null; + trace: Array<{ + step: number; + action: string; + description: string; + timestamp: number; + duration_ms?: number; + input: Record; + output: Record | null; + what: string; + why: string; + where: string; + how: string; + when: string; + status: string; + error?: string; + }>; + }>(`/api/az/admin/crawl-traces/${traceId}`); + } + + // ============================================================ + // ORCHESTRATOR ADMIN API + // ============================================================ + + async getOrchestratorMetrics() { + return this.request<{ + total_products: number; + total_brands: number; + total_stores: number; + market_sentiment: string; + market_direction: string; + healthy_count: number; + sandbox_count: number; + needs_manual_count: number; + failing_count: number; + }>('/api/admin/orchestrator/metrics'); + } + + async getOrchestratorStates() { + return this.request<{ + states: Array<{ + state: string; + storeCount: number; + }>; + }>('/api/admin/orchestrator/states'); + } + + async getOrchestratorStores(params?: { state?: string; limit?: number; offset?: number }) { + const searchParams = new URLSearchParams(); + if (params?.state && params.state !== 'all') searchParams.append('state', params.state); + if (params?.limit) searchParams.append('limit', params.limit.toString()); + if (params?.offset) searchParams.append('offset', params.offset.toString()); + const queryString = searchParams.toString() ? `?${searchParams.toString()}` : ''; + return this.request<{ + stores: Array<{ + id: number; + name: string; + city: string; + state: string; + provider: string; + platformDispensaryId: string | null; + status: string; + profileId: number | null; + profileKey: string | null; + sandboxAttempts: number; + nextRetryAt: string | null; + lastCrawlAt: string | null; + lastSuccessAt: string | null; + lastFailureAt: string | null; + failedAt: string | null; + consecutiveFailures: number; + productCount: number; + }>; + total: number; + limit: number; + offset: number; + }>(`/api/admin/orchestrator/stores${queryString}`); + } + + async getOrchestratorDispensaryProfile(dispensaryId: number) { + return this.request<{ + dispensaryId: number; + dispensaryName: string; + hasProfile: boolean; + activeProfileId: number | null; + menuType?: string; + platformDispensaryId?: string; + profile?: { + id: number; + profileKey: string; + profileName: string; + platform: string; + version: number; + status: string; + config: Record; + enabled: boolean; + sandboxAttemptCount: number; + nextRetryAt: string | null; + createdAt: string; + updatedAt: string; + }; + }>(`/api/admin/orchestrator/dispensaries/${dispensaryId}/profile`); + } + + async getOrchestratorCrawlerModule(dispensaryId: number) { + return this.request<{ + hasModule: boolean; + profileKey?: string; + platform?: string; + fileName?: string; + filePath?: string; + content?: string; + lines?: number; + error?: string; + expectedPath?: string; + }>(`/api/admin/orchestrator/dispensaries/${dispensaryId}/crawler-module`); + } + + async getOrchestratorDispensaryTraceLatest(dispensaryId: number) { + return this.request<{ + id: number; + dispensaryId: number; + runId: string; + profileId: number | null; + profileKey: string | null; + crawlerModule: string | null; + stateAtStart: string; + stateAtEnd: string; + totalSteps: number; + durationMs: number; + success: boolean; + errorMessage: string | null; + productsFound: number; + startedAt: string; + completedAt: string | null; + trace: Array<{ + step: number; + action: string; + description: string; + timestamp: number; + duration_ms?: number; + input: Record; + output: Record | null; + what: string; + why: string; + where: string; + how: string; + when: string; + status: string; + error?: string; + }>; + }>(`/api/admin/orchestrator/dispensaries/${dispensaryId}/crawl-trace/latest`); + } + + async getOrchestratorDispensaryTraces(dispensaryId: number, params?: { limit?: number; offset?: number }) { + const searchParams = new URLSearchParams(); + if (params?.limit) searchParams.append('limit', params.limit.toString()); + if (params?.offset) searchParams.append('offset', params.offset.toString()); + const queryString = searchParams.toString() ? `?${searchParams.toString()}` : ''; + return this.request<{ + traces: Array<{ + id: number; + dispensaryId: number; + runId: string; + profileKey: string | null; + stateAtStart: string; + stateAtEnd: string; + totalSteps: number; + durationMs: number; + success: boolean; + errorMessage: string | null; + productsFound: number; + startedAt: string; + completedAt: string | null; + }>; + total: number; + }>(`/api/admin/orchestrator/dispensaries/${dispensaryId}/crawl-traces${queryString}`); + } + + async getOrchestratorTraceById(traceId: number) { + return this.request<{ + id: number; + dispensaryId: number; + runId: string; + profileId: number | null; + profileKey: string | null; + crawlerModule: string | null; + stateAtStart: string; + stateAtEnd: string; + totalSteps: number; + durationMs: number; + success: boolean; + errorMessage: string | null; + productsFound: number; + startedAt: string; + completedAt: string | null; + trace: Array<{ + step: number; + action: string; + description: string; + timestamp: number; + duration_ms?: number; + input: Record; + output: Record | null; + what: string; + why: string; + where: string; + how: string; + when: string; + status: string; + error?: string; + }>; + }>(`/api/admin/orchestrator/crawl-traces/${traceId}`); + } + + // Orchestrator Crawler Control + async triggerOrchestratorCrawl(dispensaryId: number, options?: { mode?: 'sandbox' | 'production' }) { + const params = new URLSearchParams(); + if (options?.mode) params.append('mode', options.mode); + const queryString = params.toString() ? `?${params.toString()}` : ''; + return this.request<{ + success: boolean; + message: string; + jobId?: number; + }>(`/api/admin/orchestrator/crawl/${dispensaryId}${queryString}`, { + method: 'POST', + }); + } + + async updateOrchestratorAutopromote(dispensaryId: number, enabled: boolean) { + return this.request<{ + success: boolean; + message: string; + }>(`/api/admin/orchestrator/dispensaries/${dispensaryId}/autopromote`, { + method: 'PUT', + body: JSON.stringify({ allowAutopromote: enabled }), + }); + } + + // Chains API + async getOrchestratorChains() { + return this.request<{ + chains: Array<{ + id: number; + name: string; + stateCount: number; + storeCount: number; + productCount: number; + }>; + }>('/api/admin/orchestrator/chains'); + } + + // Intelligence API + async getIntelligenceBrands(params?: { limit?: number; offset?: number }) { + const searchParams = new URLSearchParams(); + if (params?.limit) searchParams.append('limit', params.limit.toString()); + if (params?.offset) searchParams.append('offset', params.offset.toString()); + const queryString = searchParams.toString() ? `?${searchParams.toString()}` : ''; + return this.request<{ + brands: Array<{ + brandName: string; + states: string[]; + storeCount: number; + skuCount: number; + avgPriceRec: number | null; + avgPriceMed: number | null; + }>; + total: number; + }>(`/api/admin/intelligence/brands${queryString}`); + } + + async getIntelligencePricing() { + return this.request<{ + byCategory: Array<{ + category: string; + avgPrice: number; + minPrice: number; + maxPrice: number; + medianPrice: number; + productCount: number; + }>; + overall: { + avgPrice: number; + minPrice: number; + maxPrice: number; + totalProducts: number; + }; + }>('/api/admin/intelligence/pricing'); + } + + async getIntelligenceStoreActivity(params?: { state?: string; chainId?: number; limit?: number }) { + const searchParams = new URLSearchParams(); + if (params?.state) searchParams.append('state', params.state); + if (params?.chainId) searchParams.append('chainId', params.chainId.toString()); + if (params?.limit) searchParams.append('limit', params.limit.toString()); + const queryString = searchParams.toString() ? `?${searchParams.toString()}` : ''; + return this.request<{ + stores: Array<{ + id: number; + name: string; + state: string; + city: string; + chainName: string | null; + skuCount: number; + snapshotCount: number; + lastCrawl: string | null; + crawlFrequencyHours: number | null; + }>; + total: number; + }>(`/api/admin/intelligence/stores${queryString}`); + } + + async getSyncInfo() { + return this.request<{ + lastEtlRun: string | null; + rowsImported: number | null; + etlStatus: string; + envVars: { + cannaiqDbConfigured: boolean; + snapshotDbConfigured: boolean; + }; + }>('/api/admin/intelligence/sync-info'); + } + + // ============================================================ + // DISCOVERY API + // ============================================================ + + async getDiscoveryStats() { + return this.request<{ + cities: { + total: number; + crawledLast24h: number; + neverCrawled: number; + }; + locations: { + total: number; + discovered: number; + verified: number; + rejected: number; + merged: number; + byState: Array<{ stateCode: string; count: number }>; + }; + }>('/api/discovery/stats'); + } + + async getDiscoveryLocations(params?: { + status?: string; + stateCode?: string; + countryCode?: string; + city?: string; + search?: string; + hasDispensary?: boolean; + limit?: number; + offset?: number; + }) { + const searchParams = new URLSearchParams(); + if (params?.status) searchParams.append('status', params.status); + if (params?.stateCode) searchParams.append('stateCode', params.stateCode); + if (params?.countryCode) searchParams.append('countryCode', params.countryCode); + if (params?.city) searchParams.append('city', params.city); + if (params?.search) searchParams.append('search', params.search); + if (params?.hasDispensary !== undefined) searchParams.append('hasDispensary', String(params.hasDispensary)); + if (params?.limit) searchParams.append('limit', params.limit.toString()); + if (params?.offset) searchParams.append('offset', params.offset.toString()); + const queryString = searchParams.toString() ? `?${searchParams.toString()}` : ''; + return this.request<{ + locations: Array<{ + id: number; + platform: string; + platformLocationId: string; + platformSlug: string; + platformMenuUrl: string; + name: string; + rawAddress: string | null; + addressLine1: string | null; + city: string | null; + stateCode: string | null; + postalCode: string | null; + countryCode: string | null; + latitude: number | null; + longitude: number | null; + status: string; + dispensaryId: number | null; + dispensaryName: string | null; + offersDelivery: boolean | null; + offersPickup: boolean | null; + isRecreational: boolean | null; + isMedical: boolean | null; + firstSeenAt: string; + lastSeenAt: string; + verifiedAt: string | null; + verifiedBy: string | null; + }>; + total: number; + limit: number; + offset: number; + }>(`/api/discovery/locations${queryString}`); + } + + async getDiscoveryLocation(id: number) { + return this.request<{ + id: number; + platform: string; + platformLocationId: string; + platformSlug: string; + platformMenuUrl: string; + name: string; + rawAddress: string | null; + addressLine1: string | null; + city: string | null; + stateCode: string | null; + postalCode: string | null; + countryCode: string | null; + latitude: number | null; + longitude: number | null; + status: string; + dispensaryId: number | null; + dispensaryName: string | null; + offersDelivery: boolean | null; + offersPickup: boolean | null; + isRecreational: boolean | null; + isMedical: boolean | null; + firstSeenAt: string; + lastSeenAt: string; + verifiedAt: string | null; + verifiedBy: string | null; + metadata: Record | null; + notes: string | null; + }>(`/api/discovery/locations/${id}`); + } + + async getDiscoveryCities(params?: { + stateCode?: string; + countryCode?: string; + crawlEnabled?: boolean; + limit?: number; + offset?: number; + }) { + const searchParams = new URLSearchParams(); + if (params?.stateCode) searchParams.append('stateCode', params.stateCode); + if (params?.countryCode) searchParams.append('countryCode', params.countryCode); + if (params?.crawlEnabled !== undefined) searchParams.append('crawlEnabled', String(params.crawlEnabled)); + if (params?.limit) searchParams.append('limit', params.limit.toString()); + if (params?.offset) searchParams.append('offset', params.offset.toString()); + const queryString = searchParams.toString() ? `?${searchParams.toString()}` : ''; + return this.request<{ + cities: Array<{ + id: number; + platform: string; + cityName: string; + citySlug: string; + stateCode: string | null; + countryCode: string; + lastCrawledAt: string | null; + crawlEnabled: boolean; + locationCount: number | null; + actualLocationCount: number; + }>; + total: number; + limit: number; + offset: number; + }>(`/api/discovery/cities${queryString}`); + } + + async verifyDiscoveryLocation(id: number, verifiedBy?: string) { + return this.request<{ + success: boolean; + action: string; + discoveryId: number; + dispensaryId: number; + message: string; + }>(`/api/discovery/locations/${id}/verify`, { + method: 'POST', + body: JSON.stringify({ verifiedBy }), + }); + } + + async linkDiscoveryLocation(id: number, dispensaryId: number, verifiedBy?: string) { + return this.request<{ + success: boolean; + action: string; + discoveryId: number; + dispensaryId: number; + dispensaryName: string; + message: string; + }>(`/api/discovery/locations/${id}/link`, { + method: 'POST', + body: JSON.stringify({ dispensaryId, verifiedBy }), + }); + } + + async rejectDiscoveryLocation(id: number, reason?: string, verifiedBy?: string) { + return this.request<{ + success: boolean; + action: string; + discoveryId: number; + message: string; + }>(`/api/discovery/locations/${id}/reject`, { + method: 'POST', + body: JSON.stringify({ reason, verifiedBy }), + }); + } + + async unrejectDiscoveryLocation(id: number) { + return this.request<{ + success: boolean; + action: string; + discoveryId: number; + message: string; + }>(`/api/discovery/locations/${id}/unreject`, { + method: 'POST', + }); + } + + async getDiscoveryMatchCandidates(id: number) { + return this.request<{ + location: any; + candidates: Array<{ + id: number; + name: string; + city: string; + state: string; + address: string; + menuType: string | null; + platformDispensaryId: string | null; + menuUrl: string | null; + matchType: string; + distanceMiles: number | null; + }>; + }>(`/api/discovery/admin/match-candidates/${id}`); + } + + async runDiscoveryState(stateCode: string, options?: { dryRun?: boolean; cityLimit?: number }) { + return this.request<{ + success: boolean; + stateCode: string; + result: any; + }>('/api/discovery/admin/discover-state', { + method: 'POST', + body: JSON.stringify({ stateCode, ...options }), + }); + } + + async runDiscoveryCity(citySlug: string, options?: { stateCode?: string; countryCode?: string; dryRun?: boolean }) { + return this.request<{ + success: boolean; + citySlug: string; + result: any; + }>('/api/discovery/admin/discover-city', { + method: 'POST', + body: JSON.stringify({ citySlug, ...options }), + }); + } + + async seedDiscoveryCities(stateCode: string) { + return this.request<{ + success: boolean; + stateCode: string; + created: number; + updated: number; + }>('/api/discovery/admin/seed-cities', { + method: 'POST', + body: JSON.stringify({ stateCode }), + }); + } + + // ============================================================ + // PLATFORM DISCOVERY API + // Routes: /api/discovery/platforms/:platformSlug/* + // + // Platform Slug Mapping (trademark-safe): + // dt = Dutchie + // jn = Jane (future) + // wm = Weedmaps (future) + // lf = Leafly (future) + // tz = Treez (future) + // ============================================================ + + async getPlatformDiscoverySummary(platformSlug: string = 'dt') { + return this.request<{ + success: boolean; + summary: { + total_locations: number; + discovered: number; + verified: number; + merged: number; + rejected: number; + }; + by_state: Array<{ + state_code: string; + total: number; + verified: number; + unlinked: number; + }>; + }>(`/api/discovery/platforms/${platformSlug}/summary`); + } + + async getPlatformDiscoveryLocations(platformSlug: string = 'dt', params?: { + status?: string; + state_code?: string; + country_code?: string; + unlinked_only?: boolean; + search?: string; + limit?: number; + offset?: number; + }) { + const searchParams = new URLSearchParams(); + if (params?.status) searchParams.append('status', params.status); + if (params?.state_code) searchParams.append('state_code', params.state_code); + if (params?.country_code) searchParams.append('country_code', params.country_code); + if (params?.unlinked_only) searchParams.append('unlinked_only', 'true'); + if (params?.search) searchParams.append('search', params.search); + if (params?.limit) searchParams.append('limit', params.limit.toString()); + if (params?.offset) searchParams.append('offset', params.offset.toString()); + const queryString = searchParams.toString() ? `?${searchParams.toString()}` : ''; + return this.request<{ + success: boolean; + locations: Array<{ + id: number; + platform: string; + platformLocationId: string; + platformSlug: string; + platformMenuUrl: string; + name: string; + rawAddress: string | null; + addressLine1: string | null; + city: string | null; + stateCode: string | null; + postalCode: string | null; + countryCode: string | null; + latitude: number | null; + longitude: number | null; + status: string; + dispensaryId: number | null; + dispensaryName: string | null; + offersDelivery: boolean | null; + offersPickup: boolean | null; + isRecreational: boolean | null; + isMedical: boolean | null; + firstSeenAt: string; + lastSeenAt: string; + verifiedAt: string | null; + verifiedBy: string | null; + notes: string | null; + }>; + total: number; + limit: number; + offset: number; + }>(`/api/discovery/platforms/${platformSlug}/locations${queryString}`); + } + + async getPlatformDiscoveryLocation(platformSlug: string = 'dt', id: number) { + return this.request<{ + success: boolean; + location: { + id: number; + platform: string; + platformLocationId: string; + platformSlug: string; + platformMenuUrl: string; + name: string; + rawAddress: string | null; + addressLine1: string | null; + addressLine2: string | null; + city: string | null; + stateCode: string | null; + postalCode: string | null; + countryCode: string | null; + latitude: number | null; + longitude: number | null; + timezone: string | null; + status: string; + dispensaryId: number | null; + dispensaryName: string | null; + dispensaryMenuUrl: string | null; + offersDelivery: boolean | null; + offersPickup: boolean | null; + isRecreational: boolean | null; + isMedical: boolean | null; + firstSeenAt: string; + lastSeenAt: string; + verifiedAt: string | null; + verifiedBy: string | null; + notes: string | null; + metadata: Record | null; + }; + }>(`/api/discovery/platforms/${platformSlug}/locations/${id}`); + } + + async verifyCreatePlatformLocation(platformSlug: string = 'dt', id: number, verifiedBy?: string) { + return this.request<{ + success: boolean; + action: string; + discoveryId: number; + dispensaryId: number; + message: string; + }>(`/api/discovery/platforms/${platformSlug}/locations/${id}/verify-create`, { + method: 'POST', + body: JSON.stringify({ verifiedBy }), + }); + } + + async verifyLinkPlatformLocation(platformSlug: string = 'dt', id: number, dispensaryId: number, verifiedBy?: string) { + return this.request<{ + success: boolean; + action: string; + discoveryId: number; + dispensaryId: number; + dispensaryName: string; + message: string; + }>(`/api/discovery/platforms/${platformSlug}/locations/${id}/verify-link`, { + method: 'POST', + body: JSON.stringify({ dispensaryId, verifiedBy }), + }); + } + + async rejectPlatformLocation(platformSlug: string = 'dt', id: number, reason?: string, verifiedBy?: string) { + return this.request<{ + success: boolean; + action: string; + discoveryId: number; + message: string; + }>(`/api/discovery/platforms/${platformSlug}/locations/${id}/reject`, { + method: 'POST', + body: JSON.stringify({ reason, verifiedBy }), + }); + } + + async unrejectPlatformLocation(platformSlug: string = 'dt', id: number) { + return this.request<{ + success: boolean; + action: string; + discoveryId: number; + message: string; + }>(`/api/discovery/platforms/${platformSlug}/locations/${id}/unreject`, { + method: 'POST', + }); + } + + async getPlatformLocationMatchCandidates(platformSlug: string = 'dt', id: number) { + return this.request<{ + success: boolean; + location: { + id: number; + name: string; + city: string; + stateCode: string; + }; + candidates: Array<{ + id: number; + name: string; + city: string; + state: string; + address: string; + menuType: string | null; + platformDispensaryId: string | null; + menuUrl: string | null; + matchType: string; + distanceMiles: number | null; + }>; + }>(`/api/discovery/platforms/${platformSlug}/locations/${id}/match-candidates`); + } + + async getPlatformDiscoveryCities(platformSlug: string = 'dt', params?: { + state_code?: string; + country_code?: string; + crawl_enabled?: boolean; + limit?: number; + offset?: number; + }) { + const searchParams = new URLSearchParams(); + if (params?.state_code) searchParams.append('state_code', params.state_code); + if (params?.country_code) searchParams.append('country_code', params.country_code); + if (params?.crawl_enabled !== undefined) searchParams.append('crawl_enabled', String(params.crawl_enabled)); + if (params?.limit) searchParams.append('limit', params.limit.toString()); + if (params?.offset) searchParams.append('offset', params.offset.toString()); + const queryString = searchParams.toString() ? `?${searchParams.toString()}` : ''; + return this.request<{ + success: boolean; + cities: Array<{ + id: number; + platform: string; + cityName: string; + citySlug: string; + stateCode: string | null; + countryCode: string; + lastCrawledAt: string | null; + crawlEnabled: boolean; + locationCount: number | null; + }>; + total: number; + }>(`/api/discovery/platforms/${platformSlug}/cities${queryString}`); + } + + async promotePlatformDiscoveryLocation(platformSlug: string = 'dt', id: number) { + return this.request<{ + success: boolean; + discoveryId: number; + dispensaryId: number; + crawlProfileId?: number; + scheduleUpdated?: boolean; + crawlJobCreated?: boolean; + error?: string; + }>(`/api/orchestrator/platforms/${platformSlug}/promote/${id}`, { + method: 'POST', + }); + } + + // ============================================================ + // RAW PAYLOAD / DEBUG API + // ============================================================ + + async getProductRawPayload(productId: number) { + return this.request<{ + product: { + id: number; + name: string; + dispensaryId: number; + dispensaryName: string; + rawPayload: Record | null; + metadata: Record | null; + createdAt: string; + updatedAt: string; + }; + }>(`/api/admin/debug/products/${productId}/raw-payload`); + } + + async getStoreSnapshots(dispensaryId: number, params?: { limit?: number; offset?: number }) { + const searchParams = new URLSearchParams(); + if (params?.limit) searchParams.append('limit', params.limit.toString()); + if (params?.offset) searchParams.append('offset', params.offset.toString()); + const queryString = searchParams.toString() ? `?${searchParams.toString()}` : ''; + return this.request<{ + snapshots: Array<{ + id: number; + productId: number; + productName: string; + brandName: string | null; + crawledAt: string; + stockStatus: string; + regularPrice: number | null; + salePrice: number | null; + rawPayload: Record | null; + }>; + total: number; + limit: number; + offset: number; + }>(`/api/admin/debug/stores/${dispensaryId}/snapshots${queryString}`); + } + + async getSnapshotRawPayload(snapshotId: number) { + return this.request<{ + snapshot: { + id: number; + productId: number; + productName: string; + dispensaryId: number; + dispensaryName: string; + crawledAt: string; + rawPayload: Record | null; + }; + }>(`/api/admin/debug/snapshots/${snapshotId}/raw-payload`); + } + + // Scraper Overview Dashboard + async getScraperOverview() { + return this.request<{ + kpi: { + totalProducts: number; + inStockProducts: number; + totalDispensaries: number; + crawlableDispensaries: number; + visibilityLost24h: number; + visibilityRestored24h: number; + totalVisibilityLost: number; + errors24h: number; + successfulJobs24h: number; + activeWorkers: number; + }; + workers: Array<{ + worker_name: string; + worker_role: string; + enabled: boolean; + last_status: string | null; + last_run_at: string | null; + next_run_at: string | null; + }>; + activityByHour: Array<{ + hour: string; + successful: number; + failed: number; + total: number; + }>; + productGrowth: Array<{ + day: string; + newProducts: number; + }>; + recentRuns: Array<{ + id: number; + jobName: string; + status: string; + startedAt: string; + completedAt: string | null; + itemsProcessed: number; + itemsSucceeded: number; + itemsFailed: number; + workerName: string | null; + workerRole: string | null; + visibilityLost: number; + visibilityRestored: number; + }>; + visibilityChanges: Array<{ + dispensaryId: number; + dispensaryName: string; + state: string; + lost24h: number; + restored24h: number; + latestLoss: string | null; + latestRestore: string | null; + }>; + }>('/api/dutchie-az/scraper/overview'); + } } export const api = new ApiClient(API_URL); diff --git a/cannaiq/src/pages/NationalDashboard.tsx b/cannaiq/src/pages/NationalDashboard.tsx new file mode 100644 index 00000000..ed11410f --- /dev/null +++ b/cannaiq/src/pages/NationalDashboard.tsx @@ -0,0 +1,378 @@ +/** + * National Dashboard + * + * Multi-state overview with key metrics and state comparison. + * Phase 4: Multi-State Expansion + */ + +import { useState, useEffect } from 'react'; +import { useNavigate } from 'react-router-dom'; +import { Layout } from '../components/Layout'; +import { StateBadge } from '../components/StateSelector'; +import { useStateStore } from '../store/stateStore'; +import { api } from '../lib/api'; +import { + Globe, + Store, + Package, + Tag, + TrendingUp, + TrendingDown, + DollarSign, + MapPin, + ArrowRight, + RefreshCw, + AlertCircle +} from 'lucide-react'; + +interface StateMetric { + state: string; + stateName: string; + storeCount: number; + totalProducts: number; + uniqueBrands: number; + avgPriceRec: number | string | null; + avgPriceMed?: number | string | null; + inStockProducts: number; + onSpecialProducts: number; +} + +/** + * Safe money formatter that handles null, undefined, strings, and invalid numbers + * @param value - Any value that might be a price + * @param fallback - What to show when value is not usable (default: '—') + * @returns Formatted price string like "$12.99" or the fallback + */ +function formatMoney(value: unknown, fallback = '—'): string { + if (value === null || value === undefined) { + return fallback; + } + + // Try to convert to number + const num = typeof value === 'string' ? parseFloat(value) : Number(value); + + // Check if it's a valid finite number + if (!Number.isFinite(num)) { + return fallback; + } + + return `$${num.toFixed(2)}`; +} + +interface NationalSummary { + totalStates: number; + activeStates: number; + totalStores: number; + totalProducts: number; + totalBrands: number; + avgPriceNational: number | null; + stateMetrics: StateMetric[]; +} + +function MetricCard({ + title, + value, + icon: Icon, + trend, + trendLabel, + onClick, +}: { + title: string; + value: string | number; + icon: any; + trend?: 'up' | 'down' | 'neutral'; + trendLabel?: string; + onClick?: () => void; +}) { + return ( +
+
+
+ +
+ {trend && ( +
+ {trend === 'up' ? : trend === 'down' ? : null} + {trendLabel} +
+ )} +
+
{value}
+
{title}
+
+ ); +} + +function StateRow({ metric, onClick }: { metric: StateMetric; onClick: () => void }) { + return ( + + +
+ + {metric.stateName} + + {metric.state} + +
+ + + {(metric.storeCount ?? 0).toLocaleString()} + + + {(metric.totalProducts ?? 0).toLocaleString()} + + + {(metric.uniqueBrands ?? 0).toLocaleString()} + + + {formatMoney(metric.avgPriceRec, '—') !== '—' ? ( + + {formatMoney(metric.avgPriceRec)} + + ) : ( + — + )} + + + 0 ? 'bg-orange-100 text-orange-700' : 'bg-gray-100 text-gray-600' + }`}> + {(metric.onSpecialProducts ?? 0).toLocaleString()} specials + + + + + + + ); +} + +export default function NationalDashboard() { + const navigate = useNavigate(); + const { setSelectedState } = useStateStore(); + const [loading, setLoading] = useState(true); + const [error, setError] = useState(null); + const [summary, setSummary] = useState(null); + const [refreshing, setRefreshing] = useState(false); + + const fetchData = async () => { + setLoading(true); + setError(null); + try { + const response = await api.get('/api/analytics/national/summary'); + if (response.data?.success && response.data.data) { + setSummary(response.data.data); + } else if (response.data?.totalStores !== undefined) { + // Handle direct data format + setSummary(response.data); + } + } catch (err: any) { + setError(err.message || 'Failed to load national data'); + console.error('Failed to fetch national summary:', err); + } finally { + setLoading(false); + } + }; + + useEffect(() => { + fetchData(); + }, []); + + const handleRefreshMetrics = async () => { + setRefreshing(true); + try { + await api.post('/api/admin/states/refresh-metrics'); + await fetchData(); + } catch (err) { + console.error('Failed to refresh metrics:', err); + } finally { + setRefreshing(false); + } + }; + + const handleStateClick = (stateCode: string) => { + setSelectedState(stateCode); + navigate('/dashboard'); + }; + + if (loading) { + return ( + +
+
Loading national data...
+
+ + ); + } + + if (error) { + return ( + +
+ +
{error}
+ +
+ + ); + } + + return ( + +
+ {/* Header */} +
+
+

National Dashboard

+

+ Multi-state cannabis market intelligence +

+
+
+ + +
+
+ + {/* Summary Cards */} + {summary && ( + <> +
+ + + + +
+ + {/* States Table */} +
+
+

State Overview

+

Click a state to view detailed analytics

+
+
+ + + + + + + + + + + + + + {summary.stateMetrics + .filter(m => m.storeCount > 0) + .sort((a, b) => b.totalProducts - a.totalProducts) + .map((metric) => ( + handleStateClick(metric.state)} + /> + ))} + +
+ State + + Stores + + Products + + Brands + + Avg Price + + Specials +
+
+
+ + {/* Quick Links */} +
+ + + + + +
+ + )} +
+ + ); +} diff --git a/cannaiq/src/pages/OrchestratorDashboard.tsx b/cannaiq/src/pages/OrchestratorDashboard.tsx new file mode 100644 index 00000000..35a051ed --- /dev/null +++ b/cannaiq/src/pages/OrchestratorDashboard.tsx @@ -0,0 +1,472 @@ +import { useEffect, useState } from 'react'; +import { useNavigate } from 'react-router-dom'; +import { Layout } from '../components/Layout'; +import { api } from '../lib/api'; +import { + Package, + Building2, + CheckCircle, + AlertTriangle, + XCircle, + RefreshCw, + ChevronDown, + Clock, + FileText, + Settings, + Code, + TrendingUp, + TrendingDown, + Minus, +} from 'lucide-react'; +import { StoreOrchestratorPanel } from '../components/StoreOrchestratorPanel'; + +interface OrchestratorMetrics { + total_products: number; + total_brands: number; + total_stores: number; + market_sentiment: string; + market_direction: string; + healthy_count: number; + sandbox_count: number; + needs_manual_count: number; + failing_count: number; +} + +interface StateInfo { + state: string; + storeCount: number; +} + +interface StoreInfo { + id: number; + name: string; + city: string; + state: string; + provider: string; + provider_raw?: string | null; + provider_display?: string; + platformDispensaryId: string | null; + status: string; + profileId: number | null; + profileKey: string | null; + sandboxAttempts: number; + nextRetryAt: string | null; + lastCrawlAt: string | null; + lastSuccessAt: string | null; + lastFailureAt: string | null; + failedAt: string | null; + consecutiveFailures: number; + productCount: number; +} + +export function OrchestratorDashboard() { + const navigate = useNavigate(); + const [metrics, setMetrics] = useState(null); + const [states, setStates] = useState([]); + const [stores, setStores] = useState([]); + const [totalStores, setTotalStores] = useState(0); + const [loading, setLoading] = useState(true); + const [selectedState, setSelectedState] = useState('all'); + const [autoRefresh, setAutoRefresh] = useState(true); + const [selectedStore, setSelectedStore] = useState(null); + const [panelTab, setPanelTab] = useState<'control' | 'trace' | 'profile' | 'module' | 'debug'>('control'); + + useEffect(() => { + loadData(); + + if (autoRefresh) { + const interval = setInterval(loadData, 30000); + return () => clearInterval(interval); + } + }, [autoRefresh, selectedState]); + + const loadData = async () => { + try { + const [metricsData, statesData, storesData] = await Promise.all([ + api.getOrchestratorMetrics(), + api.getOrchestratorStates(), + api.getOrchestratorStores({ state: selectedState, limit: 200 }), + ]); + + setMetrics(metricsData); + setStates(statesData.states || []); + setStores(storesData.stores || []); + setTotalStores(storesData.total || 0); + } catch (error) { + console.error('Failed to load orchestrator data:', error); + } finally { + setLoading(false); + } + }; + + const getStatusPill = (status: string) => { + switch (status) { + case 'production': + return ( + + + PRODUCTION + + ); + case 'sandbox': + return ( + + + SANDBOX + + ); + case 'needs_manual': + return ( + + + NEEDS MANUAL + + ); + case 'disabled': + return ( + + + DISABLED + + ); + case 'legacy': + return ( + + LEGACY + + ); + case 'pending': + return ( + + PENDING + + ); + default: + return ( + + {status || 'UNKNOWN'} + + ); + } + }; + + const getMarketDirectionIcon = (direction: string) => { + switch (direction) { + case 'up': + return ; + case 'down': + return ; + default: + return ; + } + }; + + const formatTimeAgo = (dateStr: string | null) => { + if (!dateStr) return '-'; + const date = new Date(dateStr); + const now = new Date(); + const diff = now.getTime() - date.getTime(); + const hours = Math.floor(diff / (1000 * 60 * 60)); + const days = Math.floor(hours / 24); + + if (days > 0) return `${days}d ago`; + if (hours > 0) return `${hours}h ago`; + const minutes = Math.floor(diff / (1000 * 60)); + if (minutes > 0) return `${minutes}m ago`; + return 'just now'; + }; + + if (loading) { + return ( + +
+
+

Loading orchestrator data...

+
+ + ); + } + + return ( + +
+ {/* Header */} +
+
+

Orchestrator Dashboard

+

+ Crawler observability and per-store monitoring +

+
+
+ + +
+
+ + {/* Metrics Cards - Clickable */} + {metrics && ( +
+
navigate('/admin/orchestrator/products')} + className="bg-white rounded-lg border border-gray-200 p-4 cursor-pointer hover:bg-blue-50 hover:border-blue-300 transition-colors" + > +
+ +
+

Products

+

{metrics.total_products.toLocaleString()}

+
+
+
+ +
navigate('/admin/orchestrator/brands')} + className="bg-white rounded-lg border border-gray-200 p-4 cursor-pointer hover:bg-purple-50 hover:border-purple-300 transition-colors" + > +
+ +
+

Brands

+

{metrics.total_brands.toLocaleString()}

+
+
+
+ +
navigate('/admin/orchestrator/stores')} + className="bg-white rounded-lg border border-gray-200 p-4 cursor-pointer hover:bg-gray-100 hover:border-gray-400 transition-colors" + > +
+ +
+

Stores

+

{metrics.total_stores.toLocaleString()}

+
+
+
+ +
navigate('/admin/orchestrator/stores?status=healthy')} + className="bg-white rounded-lg border border-green-200 p-4 cursor-pointer hover:bg-green-100 hover:border-green-400 transition-colors" + > +
+ +
+

Healthy

+

{metrics.healthy_count}

+
+
+
+ +
navigate('/admin/orchestrator/stores?status=sandbox')} + className="bg-white rounded-lg border border-yellow-200 p-4 cursor-pointer hover:bg-yellow-100 hover:border-yellow-400 transition-colors" + > +
+ +
+

Sandbox

+

{metrics.sandbox_count}

+
+
+
+ +
navigate('/admin/orchestrator/stores?status=needs_manual')} + className="bg-white rounded-lg border border-orange-200 p-4 cursor-pointer hover:bg-orange-100 hover:border-orange-400 transition-colors" + > +
+ +
+

Manual

+

{metrics.needs_manual_count}

+
+
+
+ +
navigate('/admin/orchestrator/stores?status=failing')} + className="bg-white rounded-lg border border-red-200 p-4 cursor-pointer hover:bg-red-100 hover:border-red-400 transition-colors" + > +
+ +
+

Failing

+

{metrics.failing_count}

+
+
+
+
+ )} + + {/* State Selector */} +
+ + + + Showing {stores.length} of {totalStores} stores + +
+ + {/* Two-column layout: Store table + Panel */} +
+ {/* Stores Table */} +
+
+

Stores

+
+
+ + + + + + + + + + + + + + + {stores.map((store) => ( + setSelectedStore(store)} + > + + + + + + + + + + ))} + +
NameStateProviderStatusLast SuccessLast FailureProductsActions
+
{store.name}
+
{store.city}
+ 		{store.state} + {store.provider_display || 'Menu'} + {getStatusPill(store.status)} + {formatTimeAgo(store.lastSuccessAt)} + + {formatTimeAgo(store.lastFailureAt)} + + {store.productCount.toLocaleString()} + +
+ + + + +
+
+
+
+ + {/* Side Panel */} +
+ {selectedStore ? ( + setSelectedStore(null)} + /> + ) : ( +
+
+ +
+

Select a store to view details

+

+ Click on any row or use the action buttons +

+
+ )} +
+
+
+ + ); +} diff --git a/cannaiq/src/pages/OrchestratorStores.tsx b/cannaiq/src/pages/OrchestratorStores.tsx new file mode 100644 index 00000000..b8828848 --- /dev/null +++ b/cannaiq/src/pages/OrchestratorStores.tsx @@ -0,0 +1,264 @@ +import { useEffect, useState } from 'react'; +import { useSearchParams, useNavigate } from 'react-router-dom'; +import { Layout } from '../components/Layout'; +import { api } from '../lib/api'; +import { + Building2, + ArrowLeft, + CheckCircle, + Clock, + AlertTriangle, + XCircle, + RefreshCw, +} from 'lucide-react'; + +interface StoreInfo { + id: number; + name: string; + city: string; + state: string; + provider: string; + provider_display?: string; + platformDispensaryId: string | null; + status: string; + profileId: number | null; + profileKey: string | null; + lastCrawlAt: string | null; + lastSuccessAt: string | null; + lastFailureAt: string | null; + productCount: number; +} + +const STATUS_FILTERS: Record boolean; icon: React.ReactNode; color: string }> = { + all: { + label: 'All Stores', + match: () => true, + icon: , + color: 'text-gray-600', + }, + healthy: { + label: 'Healthy', + match: (s) => s === 'production', + icon: , + color: 'text-green-600', + }, + sandbox: { + label: 'Sandbox', + match: (s) => s === 'sandbox', + icon: , + color: 'text-yellow-600', + }, + needs_manual: { + label: 'Needs Manual', + match: (s) => s === 'needs_manual', + icon: , + color: 'text-orange-600', + }, + failing: { + label: 'Failing', + match: (s) => s === 'failing' || s === 'disabled', + icon: , + color: 'text-red-600', + }, +}; + +export function OrchestratorStores() { + const [searchParams, setSearchParams] = useSearchParams(); + const navigate = useNavigate(); + const [stores, setStores] = useState([]); + const [loading, setLoading] = useState(true); + const [totalStores, setTotalStores] = useState(0); + + const statusFilter = searchParams.get('status') || 'all'; + + useEffect(() => { + loadStores(); + }, []); + + const loadStores = async () => { + try { + setLoading(true); + const data = await api.getOrchestratorStores({ limit: 500 }); + setStores(data.stores || []); + setTotalStores(data.total || 0); + } catch (error) { + console.error('Failed to load stores:', error); + } finally { + setLoading(false); + } + }; + + const filteredStores = stores.filter((store) => { + const filter = STATUS_FILTERS[statusFilter]; + return filter ? filter.match(store.status) : true; + }); + + const getStatusPill = (status: string) => { + switch (status) { + case 'production': + return ( + + + PRODUCTION + + ); + case 'sandbox': + return ( + + + SANDBOX + + ); + case 'needs_manual': + return ( + + + NEEDS MANUAL + + ); + case 'disabled': + case 'failing': + return ( + + + {status.toUpperCase()} + + ); + default: + return ( + + {status || 'LEGACY'} + + ); + } + }; + + const formatTimeAgo = (dateStr: string | null) => { + if (!dateStr) return '-'; + const date = new Date(dateStr); + const now = new Date(); + const diff = now.getTime() - date.getTime(); + const hours = Math.floor(diff / (1000 * 60 * 60)); + const days = Math.floor(hours / 24); + if (days > 0) return `${days}d ago`; + if (hours > 0) return `${hours}h ago`; + const minutes = Math.floor(diff / (1000 * 60)); + if (minutes > 0) return `${minutes}m ago`; + return 'just now'; + }; + + return ( + +
+ {/* Header */} +
+
+ +
+

+ Stores + {statusFilter !== 'all' && ( + + ({STATUS_FILTERS[statusFilter]?.label}) + + )} +

+

+ {filteredStores.length} of {totalStores} stores +

+
+
+ +
+ + {/* Status Filter Tabs */} +
+ {Object.entries(STATUS_FILTERS).map(([key, { label, icon, color }]) => ( + + ))} +
+ + {/* Stores Table */} +
+
+ + + + + + + + + + + + + + {loading ? ( + + + + ) : filteredStores.length === 0 ? ( + + + + ) : ( + filteredStores.map((store) => ( + navigate(`/admin/orchestrator?store=${store.id}`)} + > + + + + + + + + + )) + )} + +
NameCityStateProviderStatusLast SuccessProducts
+
+
+ No stores match this filter +
{store.name}{store.city}{store.state} + + {store.provider_display || store.provider || 'Menu'} + + {getStatusPill(store.status)} + {formatTimeAgo(store.lastSuccessAt)} + {store.productCount.toLocaleString()}
+
+
+
+ + ); +} diff --git a/cannaiq/src/pages/ScraperOverviewDashboard.tsx b/cannaiq/src/pages/ScraperOverviewDashboard.tsx new file mode 100644 index 00000000..e92f8ead --- /dev/null +++ b/cannaiq/src/pages/ScraperOverviewDashboard.tsx @@ -0,0 +1,485 @@ +import { useState, useEffect, useCallback } from 'react'; +import { Layout } from '../components/Layout'; +import { WorkerRoleBadge } from '../components/WorkerRoleBadge'; +import { api } from '../lib/api'; +import { + Package, + Building2, + Users, + EyeOff, + Eye, + AlertTriangle, + CheckCircle, + XCircle, + RefreshCw, + Activity, + TrendingUp, + Clock, +} from 'lucide-react'; +import { + AreaChart, + Area, + BarChart, + Bar, + XAxis, + YAxis, + CartesianGrid, + Tooltip, + ResponsiveContainer, + Legend, +} from 'recharts'; + +interface ScraperOverviewData { + kpi: { + totalProducts: number; + inStockProducts: number; + totalDispensaries: number; + crawlableDispensaries: number; + visibilityLost24h: number; + visibilityRestored24h: number; + totalVisibilityLost: number; + errors24h: number; + successfulJobs24h: number; + activeWorkers: number; + }; + workers: Array<{ + worker_name: string; + worker_role: string; + enabled: boolean; + last_status: string | null; + last_run_at: string | null; + next_run_at: string | null; + }>; + activityByHour: Array<{ + hour: string; + successful: number; + failed: number; + total: number; + }>; + productGrowth: Array<{ + day: string; + newProducts: number; + }>; + recentRuns: Array<{ + id: number; + jobName: string; + status: string; + startedAt: string; + completedAt: string | null; + itemsProcessed: number; + itemsSucceeded: number; + itemsFailed: number; + workerName: string | null; + workerRole: string | null; + visibilityLost: number; + visibilityRestored: number; + }>; + visibilityChanges: Array<{ + dispensaryId: number; + dispensaryName: string; + state: string; + lost24h: number; + restored24h: number; + latestLoss: string | null; + latestRestore: string | null; + }>; +} + +function formatRelativeTime(dateStr: string | null | undefined): string { + if (!dateStr) return '-'; + const date = new Date(dateStr); + const now = new Date(); + const diffMs = now.getTime() - date.getTime(); + const diffMins = Math.round(diffMs / 60000); + + if (diffMins < 0) { + const futureMins = Math.abs(diffMins); + if (futureMins < 60) return `in ${futureMins}m`; + if (futureMins < 1440) return `in ${Math.round(futureMins / 60)}h`; + return `in ${Math.round(futureMins / 1440)}d`; + } + + if (diffMins < 1) return 'just now'; + if (diffMins < 60) return `${diffMins}m ago`; + if (diffMins < 1440) return `${Math.round(diffMins / 60)}h ago`; + return `${Math.round(diffMins / 1440)}d ago`; +} + +function formatHour(isoDate: string): string { + const date = new Date(isoDate); + return date.toLocaleTimeString('en-US', { hour: '2-digit', hour12: true }); +} + +function formatDay(isoDate: string): string { + const date = new Date(isoDate); + return date.toLocaleDateString('en-US', { weekday: 'short', month: 'short', day: 'numeric' }); +} + +function StatusBadge({ status }: { status: string | null }) { + if (!status) return -; + + const config: Record = { + success: { bg: 'bg-green-100', text: 'text-green-700', icon: CheckCircle }, + running: { bg: 'bg-blue-100', text: 'text-blue-700', icon: Activity }, + pending: { bg: 'bg-yellow-100', text: 'text-yellow-700', icon: Clock }, + error: { bg: 'bg-red-100', text: 'text-red-700', icon: XCircle }, + partial: { bg: 'bg-orange-100', text: 'text-orange-700', icon: AlertTriangle }, + }; + + const cfg = config[status] || { bg: 'bg-gray-100', text: 'text-gray-700', icon: Clock }; + const Icon = cfg.icon; + + return ( + + + {status} + + ); +} + +interface KPICardProps { + title: string; + value: number | string; + subtitle?: string; + icon: any; + iconBg: string; + iconColor: string; + trend?: 'up' | 'down' | 'neutral'; + trendValue?: string; +} + +function KPICard({ title, value, subtitle, icon: Icon, iconBg, iconColor, trend, trendValue }: KPICardProps) { + return ( +
+
+
+

{title}

+

+ {typeof value === 'number' ? value.toLocaleString() : value} +

+ {subtitle && ( +

{subtitle}

+ )} + {trend && trendValue && ( +

+ + {trendValue} +

+ )} +
+
+ +
+
+
+ ); +} + +export function ScraperOverviewDashboard() { + const [data, setData] = useState(null); + const [loading, setLoading] = useState(true); + const [error, setError] = useState(null); + const [autoRefresh, setAutoRefresh] = useState(true); + + const fetchData = useCallback(async () => { + try { + const result = await api.getScraperOverview(); + setData(result); + setError(null); + } catch (err: any) { + setError(err.message || 'Failed to fetch scraper overview'); + console.error('Scraper overview error:', err); + } finally { + setLoading(false); + } + }, []); + + useEffect(() => { + fetchData(); + if (autoRefresh) { + const interval = setInterval(fetchData, 30000); + return () => clearInterval(interval); + } + }, [fetchData, autoRefresh]); + + if (loading) { + return ( + +
+ +
+ + ); + } + + const workerNames = data?.workers + ?.filter(w => w.worker_name) + .map(w => w.worker_name) + .slice(0, 4) + .join(', ') || 'None active'; + + // Format activity data for chart + const activityChartData = data?.activityByHour?.map(item => ({ + ...item, + hourLabel: formatHour(item.hour), + })) || []; + + // Format product growth data for chart + const growthChartData = data?.productGrowth?.map(item => ({ + ...item, + dayLabel: formatDay(item.day), + })) || []; + + return ( + +
+ {/* Header */} +
+
+

Scraper Overview

+

+ System health and crawler metrics dashboard +

+
+
+ + +
+
+ + {error && ( +
+

{error}

+
+ )} + + {/* KPI Cards - Top Row */} + {data && ( +
+ + + + + + +
+ )} + + {/* Charts Row */} + {data && ( +
+ {/* Scrape Activity Chart */} +
+

Scrape Activity (24h)

+
+ + + + + + + + + + + +
+
+ + {/* Product Growth Chart */} +
+

Product Growth (7 days)

+
+ + + + + + + + + +
+
+
+ )} + + {/* Bottom Panels */} + {data && ( +
+ {/* Recent Worker Runs */} +
+
+

Recent Worker Runs

+
+
+ + + + + + + + + + + + {data.recentRuns.map((run) => ( + + + + + + + + ))} + +
WorkerRoleStatusWhenStats
+ {run.workerName || run.jobName} + + + + + + {formatRelativeTime(run.startedAt)} + + {run.itemsProcessed} + {run.visibilityLost > 0 && ( + -{run.visibilityLost} + )} + {run.visibilityRestored > 0 && ( + +{run.visibilityRestored} + )} +
+
+
+ + {/* Recent Visibility Changes */} +
+
+

Recent Visibility Changes

+
+
+ {data.visibilityChanges.length === 0 ? ( +
+ No visibility changes in the last 24 hours +
+ ) : ( + + + + + + + + + + + {data.visibilityChanges.map((change) => ( + + + + + + + ))} + +
StoreStateLostRestored
+ {change.dispensaryName} + + {change.state} + + {change.lost24h > 0 ? ( + + + {change.lost24h} + + ) : ( + - + )} + + {change.restored24h > 0 ? ( + + + {change.restored24h} + + ) : ( + - + )} +
+ )} +
+
+
+ )} +
+ + ); +} + +export default ScraperOverviewDashboard; diff --git a/cannaiq/src/pages/WorkersDashboard.tsx b/cannaiq/src/pages/WorkersDashboard.tsx new file mode 100644 index 00000000..0c14c5ec --- /dev/null +++ b/cannaiq/src/pages/WorkersDashboard.tsx @@ -0,0 +1,498 @@ +import { useState, useEffect, useCallback } from 'react'; +import { Layout } from '../components/Layout'; +import { WorkerRoleBadge, formatScope } from '../components/WorkerRoleBadge'; +import { api } from '../lib/api'; +import { + Users, + Play, + Clock, + CheckCircle, + XCircle, + AlertTriangle, + RefreshCw, + ChevronDown, + ChevronUp, + Activity, +} from 'lucide-react'; + +interface Schedule { + id: number; + job_name: string; + description: string; + worker_name: string; + worker_role: string; + enabled: boolean; + base_interval_minutes: number; + jitter_minutes: number; + next_run_at: string | null; + last_run_at: string | null; + last_status: string | null; + job_config: any; +} + +interface RunLog { + id: number; + schedule_id: number; + job_name: string; + status: string; + started_at: string; + completed_at: string | null; + items_processed: number; + items_succeeded: number; + items_failed: number; + error_message: string | null; + metadata: any; + worker_name: string; + run_role: string; + duration_seconds?: number; +} + +interface MonitorSummary { + running_scheduled_jobs: number; + running_dispensary_crawl_jobs: number; + successful_jobs_24h: number; + failed_jobs_24h: number; + successful_crawls_24h: number; + failed_crawls_24h: number; + products_found_24h: number; + snapshots_created_24h: number; + last_job_started: string | null; + last_job_completed: string | null; + nextRuns: Schedule[]; +} + +function formatDuration(seconds: number | null | undefined): string { + if (!seconds) return '-'; + if (seconds < 60) return `${Math.round(seconds)}s`; + if (seconds < 3600) return `${Math.round(seconds / 60)}m`; + return `${Math.round(seconds / 3600)}h ${Math.round((seconds % 3600) / 60)}m`; +} + +function formatRelativeTime(dateStr: string | null | undefined): string { + if (!dateStr) return '-'; + const date = new Date(dateStr); + const now = new Date(); + const diffMs = now.getTime() - date.getTime(); + const diffMins = Math.round(diffMs / 60000); + + if (diffMins < 0) { + const futureMins = Math.abs(diffMins); + if (futureMins < 60) return `in ${futureMins}m`; + if (futureMins < 1440) return `in ${Math.round(futureMins / 60)}h`; + return `in ${Math.round(futureMins / 1440)}d`; + } + + if (diffMins < 1) return 'just now'; + if (diffMins < 60) return `${diffMins}m ago`; + if (diffMins < 1440) return `${Math.round(diffMins / 60)}h ago`; + return `${Math.round(diffMins / 1440)}d ago`; +} + +function StatusBadge({ status }: { status: string | null }) { + if (!status) return -; + + const config: Record = { + success: { bg: 'bg-green-100', text: 'text-green-700', icon: CheckCircle }, + running: { bg: 'bg-blue-100', text: 'text-blue-700', icon: Activity }, + pending: { bg: 'bg-yellow-100', text: 'text-yellow-700', icon: Clock }, + error: { bg: 'bg-red-100', text: 'text-red-700', icon: XCircle }, + partial: { bg: 'bg-orange-100', text: 'text-orange-700', icon: AlertTriangle }, + }; + + const cfg = config[status] || { bg: 'bg-gray-100', text: 'text-gray-700', icon: Clock }; + const Icon = cfg.icon; + + return ( + + + {status} + + ); +} + +export function WorkersDashboard() { + const [schedules, setSchedules] = useState([]); + const [selectedWorker, setSelectedWorker] = useState(null); + const [workerLogs, setWorkerLogs] = useState([]); + const [summary, setSummary] = useState(null); + const [loading, setLoading] = useState(true); + const [logsLoading, setLogsLoading] = useState(false); + const [error, setError] = useState(null); + const [triggering, setTriggering] = useState(null); + + const fetchData = useCallback(async () => { + try { + const [schedulesRes, summaryRes] = await Promise.all([ + api.get('/api/dutchie-az/admin/schedules'), + api.get('/api/dutchie-az/monitor/summary'), + ]); + + setSchedules(schedulesRes.data.schedules || []); + setSummary(summaryRes.data); + setError(null); + } catch (err: any) { + setError(err.message || 'Failed to fetch data'); + } finally { + setLoading(false); + } + }, []); + + const fetchWorkerLogs = useCallback(async (scheduleId: number) => { + setLogsLoading(true); + try { + const res = await api.get(`/api/dutchie-az/admin/schedules/${scheduleId}/logs?limit=20`); + setWorkerLogs(res.data.logs || []); + } catch (err: any) { + console.error('Failed to fetch worker logs:', err); + setWorkerLogs([]); + } finally { + setLogsLoading(false); + } + }, []); + + useEffect(() => { + fetchData(); + const interval = setInterval(fetchData, 30000); + return () => clearInterval(interval); + }, [fetchData]); + + useEffect(() => { + if (selectedWorker) { + fetchWorkerLogs(selectedWorker.id); + } else { + setWorkerLogs([]); + } + }, [selectedWorker, fetchWorkerLogs]); + + const handleSelectWorker = (schedule: Schedule) => { + if (selectedWorker?.id === schedule.id) { + setSelectedWorker(null); + } else { + setSelectedWorker(schedule); + } + }; + + const handleTrigger = async (scheduleId: number) => { + setTriggering(scheduleId); + try { + await api.post(`/api/dutchie-az/admin/schedules/${scheduleId}/trigger`); + // Refresh data after trigger + setTimeout(fetchData, 1000); + } catch (err: any) { + console.error('Failed to trigger schedule:', err); + } finally { + setTriggering(null); + } + }; + + if (loading) { + return ( + +
+ +
+ + ); + } + + return ( + +
+ {/* Header */} +
+
+

Crawler Workers

+

+ Named workforce dashboard - Alice, Henry, Bella, Oscar +

+
+ +
+ + {error && ( +
+

{error}

+
+ )} + + {/* Summary Cards */} + {summary && ( +
+
+
+
+ +
+
+

Running Jobs

+

+ {summary.running_scheduled_jobs + summary.running_dispensary_crawl_jobs} +

+
+
+
+
+
+
+ +
+
+

Successful (24h)

+

{summary.successful_jobs_24h}

+
+
+
+
+
+
+ +
+
+

Failed (24h)

+

{summary.failed_jobs_24h}

+
+
+
+
+
+
+ +
+
+

Active Workers

+

{schedules.filter(s => s.enabled).length}

+
+
+
+
+ )} + + {/* Workers Table */} +
+
+

Workers

+
+ + + + + + + + + + + + + + {schedules.map((schedule) => ( + handleSelectWorker(schedule)} + > + + + + + + + + + ))} + +
+ Worker + + Role + + Scope + + Last Run + + Next Run + + Status + + Actions +
+
+ + + {schedule.worker_name || schedule.job_name} + + {selectedWorker?.id === schedule.id ? ( + + ) : ( + + )} +
+ {schedule.description && ( +

{schedule.description}

+ )} + 		+ + + {formatScope(schedule.job_config)} + + {formatRelativeTime(schedule.last_run_at)} + + {schedule.enabled ? formatRelativeTime(schedule.next_run_at) : 'disabled'} + + + + +
+
+ + {/* Worker Detail Pane */} + {selectedWorker && ( +
+
+
+
+

+ {selectedWorker.worker_name || selectedWorker.job_name} +

+ +
+
+ Scope: {formatScope(selectedWorker.job_config)} +
+
+ {selectedWorker.description && ( +

{selectedWorker.description}

+ )} +
+ + {/* Run History */} +
+

Recent Run History

+ {logsLoading ? ( +
+ +
+ ) : workerLogs.length === 0 ? ( +

No run history available

+ ) : ( + + + + + + + + + + + + + {workerLogs.map((log) => { + const duration = log.completed_at + ? (new Date(log.completed_at).getTime() - + new Date(log.started_at).getTime()) / + 1000 + : null; + const visLost = log.metadata?.visibilityLostCount; + const visRestored = log.metadata?.visibilityRestoredCount; + + return ( + + + + + + + + + ); + })} + +
+ Started + + Duration + + Status + + Processed + + Visibility Stats + + Error +
+ {formatRelativeTime(log.started_at)} + + {formatDuration(duration)} + + + + {log.items_succeeded} + / + {log.items_processed} + {log.items_failed > 0 && ( + <> + ( + {log.items_failed} failed + ) + + )} + + {visLost !== undefined || visRestored !== undefined ? ( + + {visLost !== undefined && visLost > 0 && ( + + -{visLost} lost + + )} + {visRestored !== undefined && visRestored > 0 && ( + + +{visRestored} restored + + )} + {visLost === 0 && visRestored === 0 && ( + no changes + )} + + ) : ( + - + )} + + {log.error_message || '-'} +
+ )} +
+
+ )} +
+ + ); +} + +export default WorkersDashboard;