feat(cannaiq): Add Workers Dashboard and visibility tracking
Workers Dashboard: - New /workers route with two-pane layout - Workers table showing Alice, Henry, Bella, Oscar with role badges - Run history with visibility stats (lost/restored counts) - "Run Now" action to trigger workers immediately Migrations: - 057: Add visibility tracking columns (visibility_lost, visibility_lost_at, visibility_restored_at) - 058: Add ID resolution columns for Henry worker - 059: Add job queue columns (max_retries, retry_count, worker_id, locked_at, locked_by) Backend fixes: - Add httpStatus to CrawlResult interface for error classification - Fix pool.ts typing for event listener - Update completeJob to accept visibility stats in metadata Frontend fixes: - Fix NationalDashboard crash with safe formatMoney helper - Fix OrchestratorDashboard/Stores StoreInfo type mismatches - Add workerName/workerRole to getDutchieAZSchedules API type 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
Kelly
64
backend/migrations/057_visibility_tracking_columns.sql
Normal file
64
backend/migrations/057_visibility_tracking_columns.sql
Normal file
@@ -0,0 +1,64 @@
|
||||
-- Migration 057: Add visibility tracking columns to dutchie_products
|
||||
--
|
||||
-- Supports Bella (Product Sync) worker visibility-loss tracking:
|
||||
-- - visibility_lost: TRUE when product disappears from GraphQL feed
|
||||
-- - visibility_lost_at: Timestamp when product first went missing
|
||||
-- - visibility_restored_at: Timestamp when product reappeared
|
||||
--
|
||||
-- These columns enable tracking of products that temporarily or permanently
|
||||
-- disappear from Dutchie GraphQL API responses.
|
||||
|
||||
-- ============================================================
|
||||
-- 1. ADD VISIBILITY TRACKING COLUMNS TO dutchie_products
|
||||
-- ============================================================
|
||||
|
||||
ALTER TABLE dutchie_products
|
||||
ADD COLUMN IF NOT EXISTS visibility_lost BOOLEAN DEFAULT FALSE,
|
||||
ADD COLUMN IF NOT EXISTS visibility_lost_at TIMESTAMPTZ,
|
||||
ADD COLUMN IF NOT EXISTS visibility_restored_at TIMESTAMPTZ;
|
||||
|
||||
COMMENT ON COLUMN dutchie_products.visibility_lost IS 'TRUE when product is missing from GraphQL feed';
|
||||
COMMENT ON COLUMN dutchie_products.visibility_lost_at IS 'Timestamp when product first went missing from feed';
|
||||
COMMENT ON COLUMN dutchie_products.visibility_restored_at IS 'Timestamp when product reappeared after being missing';
|
||||
|
||||
-- ============================================================
|
||||
-- 2. CREATE INDEX FOR VISIBILITY QUERIES
|
||||
-- ============================================================
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_dutchie_products_visibility_lost
|
||||
ON dutchie_products(visibility_lost)
|
||||
WHERE visibility_lost = TRUE;
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_dutchie_products_visibility_lost_at
|
||||
ON dutchie_products(visibility_lost_at)
|
||||
WHERE visibility_lost_at IS NOT NULL;
|
||||
|
||||
-- ============================================================
|
||||
-- 3. CREATE VIEW FOR VISIBILITY ANALYTICS
|
||||
-- ============================================================
|
||||
|
||||
CREATE OR REPLACE VIEW v_visibility_summary AS
|
||||
SELECT
|
||||
d.id AS dispensary_id,
|
||||
d.name AS dispensary_name,
|
||||
d.state,
|
||||
COUNT(dp.id) AS total_products,
|
||||
COUNT(dp.id) FILTER (WHERE dp.visibility_lost = TRUE) AS visibility_lost_count,
|
||||
COUNT(dp.id) FILTER (WHERE dp.visibility_lost = FALSE OR dp.visibility_lost IS NULL) AS visible_count,
|
||||
COUNT(dp.id) FILTER (WHERE dp.visibility_restored_at IS NOT NULL) AS restored_count,
|
||||
MAX(dp.visibility_lost_at) AS latest_loss_at,
|
||||
MAX(dp.visibility_restored_at) AS latest_restore_at
|
||||
FROM dispensaries d
|
||||
LEFT JOIN dutchie_products dp ON d.id = dp.dispensary_id
|
||||
WHERE d.menu_type = 'dutchie'
|
||||
GROUP BY d.id, d.name, d.state;
|
||||
|
||||
COMMENT ON VIEW v_visibility_summary IS 'Aggregated visibility metrics per dispensary for dashboard analytics';
|
||||
|
||||
-- ============================================================
|
||||
-- 4. RECORD MIGRATION
|
||||
-- ============================================================
|
||||
|
||||
INSERT INTO schema_migrations (version, name, applied_at)
|
||||
VALUES (57, '057_visibility_tracking_columns', NOW())
|
||||
ON CONFLICT (version) DO NOTHING;
|
||||
46
backend/migrations/058_add_id_resolution_columns.sql
Normal file
46
backend/migrations/058_add_id_resolution_columns.sql
Normal file
@@ -0,0 +1,46 @@
|
||||
-- Migration 058: Add ID resolution tracking columns to dispensaries
|
||||
--
|
||||
-- Supports Henry (Entry Point Finder) worker tracking:
|
||||
-- - id_resolution_attempts: Count of how many times we've tried to resolve platform ID
|
||||
-- - last_id_resolution_at: When we last tried (matches code expectation)
|
||||
-- - id_resolution_status: Current status (pending, resolved, failed)
|
||||
-- - id_resolution_error: Last error message from resolution attempt
|
||||
|
||||
-- ============================================================
|
||||
-- 1. ADD ID RESOLUTION COLUMNS TO dispensaries
|
||||
-- ============================================================
|
||||
|
||||
ALTER TABLE dispensaries
|
||||
ADD COLUMN IF NOT EXISTS id_resolution_attempts INTEGER DEFAULT 0,
|
||||
ADD COLUMN IF NOT EXISTS last_id_resolution_at TIMESTAMPTZ,
|
||||
ADD COLUMN IF NOT EXISTS id_resolution_status VARCHAR(20) DEFAULT 'pending',
|
||||
ADD COLUMN IF NOT EXISTS id_resolution_error TEXT;
|
||||
|
||||
COMMENT ON COLUMN dispensaries.id_resolution_attempts IS 'Number of attempts to resolve platform_dispensary_id';
|
||||
COMMENT ON COLUMN dispensaries.last_id_resolution_at IS 'Timestamp of last ID resolution attempt';
|
||||
COMMENT ON COLUMN dispensaries.id_resolution_status IS 'Status: pending, resolved, failed';
|
||||
COMMENT ON COLUMN dispensaries.id_resolution_error IS 'Last error message from ID resolution attempt';
|
||||
|
||||
-- Additional columns needed by worker/scheduler
|
||||
ALTER TABLE dispensaries
|
||||
ADD COLUMN IF NOT EXISTS failed_at TIMESTAMPTZ,
|
||||
ADD COLUMN IF NOT EXISTS failure_notes TEXT;
|
||||
|
||||
COMMENT ON COLUMN dispensaries.failed_at IS 'Timestamp when dispensary was marked as permanently failed';
|
||||
COMMENT ON COLUMN dispensaries.failure_notes IS 'Notes about why dispensary was marked as failed';
|
||||
|
||||
-- ============================================================
|
||||
-- 2. CREATE INDEX FOR RESOLUTION QUERIES
|
||||
-- ============================================================
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_dispensaries_id_resolution_status
|
||||
ON dispensaries(id_resolution_status)
|
||||
WHERE id_resolution_status = 'pending';
|
||||
|
||||
-- ============================================================
|
||||
-- 3. RECORD MIGRATION
|
||||
-- ============================================================
|
||||
|
||||
INSERT INTO schema_migrations (version, name, applied_at)
|
||||
VALUES (58, '058_add_id_resolution_columns', NOW())
|
||||
ON CONFLICT (version) DO NOTHING;
|
||||
67
backend/migrations/059_job_queue_columns.sql
Normal file
67
backend/migrations/059_job_queue_columns.sql
Normal file
@@ -0,0 +1,67 @@
|
||||
-- Migration 059: Add missing columns to dispensary_crawl_jobs
|
||||
--
|
||||
-- Required for worker job processing:
|
||||
-- - max_retries: Maximum retry attempts for a job
|
||||
-- - retry_count: Current retry count
|
||||
-- - worker_id: ID of worker processing the job
|
||||
-- - locked_at: When the job was locked by a worker
|
||||
-- - locked_by: Hostname of worker that locked the job
|
||||
|
||||
-- ============================================================
|
||||
-- 1. ADD JOB QUEUE COLUMNS
|
||||
-- ============================================================
|
||||
|
||||
ALTER TABLE dispensary_crawl_jobs
|
||||
ADD COLUMN IF NOT EXISTS max_retries INTEGER DEFAULT 3,
|
||||
ADD COLUMN IF NOT EXISTS retry_count INTEGER DEFAULT 0,
|
||||
ADD COLUMN IF NOT EXISTS worker_id VARCHAR(100),
|
||||
ADD COLUMN IF NOT EXISTS locked_at TIMESTAMPTZ,
|
||||
ADD COLUMN IF NOT EXISTS locked_by VARCHAR(100),
|
||||
ADD COLUMN IF NOT EXISTS updated_at TIMESTAMPTZ DEFAULT NOW();
|
||||
|
||||
COMMENT ON COLUMN dispensary_crawl_jobs.max_retries IS 'Maximum number of retry attempts';
|
||||
COMMENT ON COLUMN dispensary_crawl_jobs.retry_count IS 'Current retry count';
|
||||
COMMENT ON COLUMN dispensary_crawl_jobs.worker_id IS 'ID of worker processing this job';
|
||||
COMMENT ON COLUMN dispensary_crawl_jobs.locked_at IS 'When job was locked by worker';
|
||||
COMMENT ON COLUMN dispensary_crawl_jobs.locked_by IS 'Hostname of worker that locked job';
|
||||
|
||||
-- ============================================================
|
||||
-- 2. CREATE INDEXES FOR JOB QUEUE QUERIES
|
||||
-- ============================================================
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_crawl_jobs_status_priority
|
||||
ON dispensary_crawl_jobs(status, priority DESC)
|
||||
WHERE status = 'pending';
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_crawl_jobs_worker_id
|
||||
ON dispensary_crawl_jobs(worker_id)
|
||||
WHERE worker_id IS NOT NULL;
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_crawl_jobs_locked_at
|
||||
ON dispensary_crawl_jobs(locked_at)
|
||||
WHERE locked_at IS NOT NULL;
|
||||
|
||||
-- ============================================================
|
||||
-- 3. CREATE QUEUE STATS VIEW
|
||||
-- ============================================================
|
||||
|
||||
CREATE OR REPLACE VIEW v_queue_stats AS
|
||||
SELECT
|
||||
COUNT(*) FILTER (WHERE status = 'pending') AS pending_jobs,
|
||||
COUNT(*) FILTER (WHERE status = 'running') AS running_jobs,
|
||||
COUNT(*) FILTER (WHERE status = 'completed' AND completed_at > NOW() - INTERVAL '1 hour') AS completed_1h,
|
||||
COUNT(*) FILTER (WHERE status = 'failed' AND completed_at > NOW() - INTERVAL '1 hour') AS failed_1h,
|
||||
COUNT(DISTINCT worker_id) FILTER (WHERE status = 'running') AS active_workers,
|
||||
ROUND((AVG(EXTRACT(EPOCH FROM (completed_at - started_at)))
|
||||
FILTER (WHERE status = 'completed' AND completed_at > NOW() - INTERVAL '1 hour'))::numeric, 2) AS avg_duration_seconds
|
||||
FROM dispensary_crawl_jobs;
|
||||
|
||||
COMMENT ON VIEW v_queue_stats IS 'Real-time queue statistics for monitoring dashboard';
|
||||
|
||||
-- ============================================================
|
||||
-- 4. RECORD MIGRATION
|
||||
-- ============================================================
|
||||
|
||||
INSERT INTO schema_migrations (version, name, applied_at)
|
||||
VALUES (59, '059_job_queue_columns', NOW())
|
||||
ON CONFLICT (version) DO NOTHING;
|
||||
94
backend/src/db/pool.ts
Normal file
94
backend/src/db/pool.ts
Normal file
@@ -0,0 +1,94 @@
|
||||
/**
|
||||
* Runtime Database Pool
|
||||
*
|
||||
* This is the canonical database pool for all runtime services.
|
||||
* Import pool from here, NOT from migrate.ts.
|
||||
*
|
||||
* migrate.ts is for CLI migrations only and must NOT be imported at runtime.
|
||||
*/
|
||||
|
||||
import dotenv from 'dotenv';
|
||||
import { Pool } from 'pg';
|
||||
|
||||
// Load .env before any env var access
|
||||
dotenv.config();
|
||||
|
||||
/**
|
||||
* Get the database connection string from environment variables.
|
||||
* Supports both CANNAIQ_DB_URL and individual CANNAIQ_DB_* vars.
|
||||
*/
|
||||
function getConnectionString(): string {
|
||||
// Priority 1: Full connection URL
|
||||
if (process.env.CANNAIQ_DB_URL) {
|
||||
return process.env.CANNAIQ_DB_URL;
|
||||
}
|
||||
|
||||
// Priority 2: Build from individual env vars
|
||||
const host = process.env.CANNAIQ_DB_HOST;
|
||||
const port = process.env.CANNAIQ_DB_PORT;
|
||||
const name = process.env.CANNAIQ_DB_NAME;
|
||||
const user = process.env.CANNAIQ_DB_USER;
|
||||
const pass = process.env.CANNAIQ_DB_PASS;
|
||||
|
||||
// Check if all individual vars are present
|
||||
if (host && port && name && user && pass) {
|
||||
return `postgresql://${user}:${pass}@${host}:${port}/${name}`;
|
||||
}
|
||||
|
||||
// Fallback: Try DATABASE_URL for legacy compatibility
|
||||
if (process.env.DATABASE_URL) {
|
||||
return process.env.DATABASE_URL;
|
||||
}
|
||||
|
||||
// Report what's missing
|
||||
const required = ['CANNAIQ_DB_HOST', 'CANNAIQ_DB_PORT', 'CANNAIQ_DB_NAME', 'CANNAIQ_DB_USER', 'CANNAIQ_DB_PASS'];
|
||||
const missing = required.filter((key) => !process.env[key]);
|
||||
|
||||
throw new Error(
|
||||
`[DB Pool] Missing database configuration.\n` +
|
||||
`Set CANNAIQ_DB_URL, or all of: ${missing.join(', ')}`
|
||||
);
|
||||
}
|
||||
|
||||
// Lazy-initialized pool singleton
|
||||
let _pool: Pool | null = null;
|
||||
|
||||
/**
|
||||
* Get the database pool (lazy singleton)
|
||||
*/
|
||||
export function getPool(): Pool {
|
||||
if (!_pool) {
|
||||
_pool = new Pool({
|
||||
connectionString: getConnectionString(),
|
||||
max: 10,
|
||||
idleTimeoutMillis: 30000,
|
||||
connectionTimeoutMillis: 5000,
|
||||
});
|
||||
|
||||
_pool.on('error', (err) => {
|
||||
console.error('[DB Pool] Unexpected error on idle client:', err);
|
||||
});
|
||||
}
|
||||
return _pool;
|
||||
}
|
||||
|
||||
/**
|
||||
* The database pool for runtime use.
|
||||
* This is a getter that lazily initializes on first access.
|
||||
*/
|
||||
export const pool = {
|
||||
query: (...args: Parameters<Pool['query']>) => getPool().query(...args),
|
||||
connect: () => getPool().connect(),
|
||||
end: () => getPool().end(),
|
||||
on: (event: 'error' | 'connect' | 'acquire' | 'remove' | 'release', listener: (...args: any[]) => void) => getPool().on(event as any, listener),
|
||||
};
|
||||
|
||||
/**
|
||||
* Close the pool connection
|
||||
*/
|
||||
export async function closePool(): Promise<void> {
|
||||
if (_pool) {
|
||||
await _pool.end();
|
||||
_pool = null;
|
||||
}
|
||||
}
|
||||
@@ -8,6 +8,10 @@
|
||||
import { query, getClient } from '../db/connection';
|
||||
import { v4 as uuidv4 } from 'uuid';
|
||||
import * as os from 'os';
|
||||
import { DEFAULT_CONFIG } from './store-validator';
|
||||
|
||||
// Minimum gap between crawls for the same dispensary (in minutes)
|
||||
const MIN_CRAWL_GAP_MINUTES = DEFAULT_CONFIG.minCrawlGapMinutes; // 2 minutes
|
||||
|
||||
// ============================================================
|
||||
// TYPES
|
||||
@@ -97,11 +101,30 @@ export function getWorkerHostname(): string {
|
||||
// JOB ENQUEUEING
|
||||
// ============================================================
|
||||
|
||||
export interface EnqueueResult {
|
||||
jobId: number | null;
|
||||
skipped: boolean;
|
||||
reason?: 'already_queued' | 'too_soon' | 'error';
|
||||
message?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Enqueue a new job for processing
|
||||
* Returns null if a pending/running job already exists for this dispensary
|
||||
* or if a job was completed/failed within the minimum gap period
|
||||
*/
|
||||
export async function enqueueJob(options: EnqueueJobOptions): Promise<number | null> {
|
||||
const result = await enqueueJobWithReason(options);
|
||||
return result.jobId;
|
||||
}
|
||||
|
||||
/**
|
||||
* Enqueue a new job with detailed result info
|
||||
* Enforces:
|
||||
* 1. No duplicate pending/running jobs for same dispensary
|
||||
* 2. Minimum 2-minute gap between crawls for same dispensary
|
||||
*/
|
||||
export async function enqueueJobWithReason(options: EnqueueJobOptions): Promise<EnqueueResult> {
|
||||
const {
|
||||
jobType,
|
||||
dispensaryId,
|
||||
@@ -121,31 +144,87 @@ export async function enqueueJob(options: EnqueueJobOptions): Promise<number | n
|
||||
|
||||
if (existing.length > 0) {
|
||||
console.log(`[JobQueue] Skipping enqueue - job already exists for dispensary ${dispensaryId}`);
|
||||
return null;
|
||||
return {
|
||||
jobId: null,
|
||||
skipped: true,
|
||||
reason: 'already_queued',
|
||||
message: `Job already pending/running for dispensary ${dispensaryId}`,
|
||||
};
|
||||
}
|
||||
|
||||
// Check minimum gap since last job (2 minutes)
|
||||
const { rows: recent } = await query<any>(
|
||||
`SELECT id, created_at, status
|
||||
FROM dispensary_crawl_jobs
|
||||
WHERE dispensary_id = $1
|
||||
ORDER BY created_at DESC
|
||||
LIMIT 1`,
|
||||
[dispensaryId]
|
||||
);
|
||||
|
||||
if (recent.length > 0) {
|
||||
const lastJobTime = new Date(recent[0].created_at);
|
||||
const minGapMs = MIN_CRAWL_GAP_MINUTES * 60 * 1000;
|
||||
const timeSinceLastJob = Date.now() - lastJobTime.getTime();
|
||||
|
||||
if (timeSinceLastJob < minGapMs) {
|
||||
const waitSeconds = Math.ceil((minGapMs - timeSinceLastJob) / 1000);
|
||||
console.log(`[JobQueue] Skipping enqueue - minimum ${MIN_CRAWL_GAP_MINUTES}min gap not met for dispensary ${dispensaryId}. Wait ${waitSeconds}s`);
|
||||
return {
|
||||
jobId: null,
|
||||
skipped: true,
|
||||
reason: 'too_soon',
|
||||
message: `Minimum ${MIN_CRAWL_GAP_MINUTES}-minute gap required. Try again in ${waitSeconds} seconds.`,
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const { rows } = await query<any>(
|
||||
`INSERT INTO dispensary_crawl_jobs (job_type, dispensary_id, status, priority, max_retries, metadata, created_at)
|
||||
VALUES ($1, $2, 'pending', $3, $4, $5, NOW())
|
||||
RETURNING id`,
|
||||
[jobType, dispensaryId || null, priority, maxRetries, metadata ? JSON.stringify(metadata) : null]
|
||||
);
|
||||
try {
|
||||
const { rows } = await query<any>(
|
||||
`INSERT INTO dispensary_crawl_jobs (job_type, dispensary_id, status, priority, max_retries, metadata, created_at)
|
||||
VALUES ($1, $2, 'pending', $3, $4, $5, NOW())
|
||||
RETURNING id`,
|
||||
[jobType, dispensaryId || null, priority, maxRetries, metadata ? JSON.stringify(metadata) : null]
|
||||
);
|
||||
|
||||
const jobId = rows[0].id;
|
||||
console.log(`[JobQueue] Enqueued job ${jobId} (type=${jobType}, dispensary=${dispensaryId})`);
|
||||
return jobId;
|
||||
const jobId = rows[0].id;
|
||||
console.log(`[JobQueue] Enqueued job ${jobId} (type=${jobType}, dispensary=${dispensaryId})`);
|
||||
return { jobId, skipped: false };
|
||||
} catch (error: any) {
|
||||
// Handle database trigger rejection for minimum gap
|
||||
if (error.message?.includes('Minimum') && error.message?.includes('gap')) {
|
||||
console.log(`[JobQueue] DB rejected - minimum gap not met for dispensary ${dispensaryId}`);
|
||||
return {
|
||||
jobId: null,
|
||||
skipped: true,
|
||||
reason: 'too_soon',
|
||||
message: error.message,
|
||||
};
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
export interface BulkEnqueueResult {
|
||||
enqueued: number;
|
||||
skipped: number;
|
||||
skippedReasons: {
|
||||
alreadyQueued: number;
|
||||
tooSoon: number;
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Bulk enqueue jobs for multiple dispensaries
|
||||
* Skips dispensaries that already have pending/running jobs
|
||||
* or have jobs within the minimum gap period
|
||||
*/
|
||||
export async function bulkEnqueueJobs(
|
||||
jobType: string,
|
||||
dispensaryIds: number[],
|
||||
options: { priority?: number; metadata?: Record<string, any> } = {}
|
||||
): Promise<{ enqueued: number; skipped: number }> {
|
||||
): Promise<BulkEnqueueResult> {
|
||||
const { priority = 0, metadata } = options;
|
||||
|
||||
// Get dispensaries that already have pending/running jobs
|
||||
@@ -156,11 +235,31 @@ export async function bulkEnqueueJobs(
|
||||
);
|
||||
const existingSet = new Set(existing.map((r: any) => r.dispensary_id));
|
||||
|
||||
// Filter out dispensaries with existing jobs
|
||||
const toEnqueue = dispensaryIds.filter(id => !existingSet.has(id));
|
||||
// Get dispensaries that have recent jobs within minimum gap
|
||||
const { rows: recent } = await query<any>(
|
||||
`SELECT DISTINCT dispensary_id FROM dispensary_crawl_jobs
|
||||
WHERE dispensary_id = ANY($1)
|
||||
AND created_at > NOW() - ($2 || ' minutes')::INTERVAL
|
||||
AND dispensary_id NOT IN (
|
||||
SELECT dispensary_id FROM dispensary_crawl_jobs
|
||||
WHERE dispensary_id = ANY($1) AND status IN ('pending', 'running')
|
||||
)`,
|
||||
[dispensaryIds, MIN_CRAWL_GAP_MINUTES]
|
||||
);
|
||||
const recentSet = new Set(recent.map((r: any) => r.dispensary_id));
|
||||
|
||||
// Filter out dispensaries with existing or recent jobs
|
||||
const toEnqueue = dispensaryIds.filter(id => !existingSet.has(id) && !recentSet.has(id));
|
||||
|
||||
if (toEnqueue.length === 0) {
|
||||
return { enqueued: 0, skipped: dispensaryIds.length };
|
||||
return {
|
||||
enqueued: 0,
|
||||
skipped: dispensaryIds.length,
|
||||
skippedReasons: {
|
||||
alreadyQueued: existingSet.size,
|
||||
tooSoon: recentSet.size,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
// Bulk insert - each row needs 4 params: job_type, dispensary_id, priority, metadata
|
||||
@@ -181,8 +280,15 @@ export async function bulkEnqueueJobs(
|
||||
params
|
||||
);
|
||||
|
||||
console.log(`[JobQueue] Bulk enqueued ${toEnqueue.length} jobs, skipped ${existingSet.size}`);
|
||||
return { enqueued: toEnqueue.length, skipped: existingSet.size };
|
||||
console.log(`[JobQueue] Bulk enqueued ${toEnqueue.length} jobs, skipped ${existingSet.size} (queued) + ${recentSet.size} (recent)`);
|
||||
return {
|
||||
enqueued: toEnqueue.length,
|
||||
skipped: existingSet.size + recentSet.size,
|
||||
skippedReasons: {
|
||||
alreadyQueued: existingSet.size,
|
||||
tooSoon: recentSet.size,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
// ============================================================
|
||||
@@ -311,22 +417,48 @@ export async function heartbeat(jobId: number): Promise<void> {
|
||||
|
||||
/**
|
||||
* Mark job as completed
|
||||
*
|
||||
* Stores visibility tracking stats (visibilityLostCount, visibilityRestoredCount)
|
||||
* in the metadata JSONB column for dashboard analytics.
|
||||
*/
|
||||
export async function completeJob(
|
||||
jobId: number,
|
||||
result: { productsFound?: number; productsUpserted?: number; snapshotsCreated?: number }
|
||||
result: {
|
||||
productsFound?: number;
|
||||
productsUpserted?: number;
|
||||
snapshotsCreated?: number;
|
||||
visibilityLostCount?: number;
|
||||
visibilityRestoredCount?: number;
|
||||
}
|
||||
): Promise<void> {
|
||||
// Build metadata with visibility stats if provided
|
||||
const metadata: Record<string, any> = {};
|
||||
if (result.visibilityLostCount !== undefined) {
|
||||
metadata.visibilityLostCount = result.visibilityLostCount;
|
||||
}
|
||||
if (result.visibilityRestoredCount !== undefined) {
|
||||
metadata.visibilityRestoredCount = result.visibilityRestoredCount;
|
||||
}
|
||||
if (result.snapshotsCreated !== undefined) {
|
||||
metadata.snapshotsCreated = result.snapshotsCreated;
|
||||
}
|
||||
|
||||
await query(
|
||||
`UPDATE dispensary_crawl_jobs
|
||||
SET
|
||||
status = 'completed',
|
||||
completed_at = NOW(),
|
||||
products_found = COALESCE($2, products_found),
|
||||
products_upserted = COALESCE($3, products_upserted),
|
||||
snapshots_created = COALESCE($4, snapshots_created),
|
||||
products_updated = COALESCE($3, products_updated),
|
||||
metadata = COALESCE(metadata, '{}'::jsonb) || $4::jsonb,
|
||||
updated_at = NOW()
|
||||
WHERE id = $1`,
|
||||
[jobId, result.productsFound, result.productsUpserted, result.snapshotsCreated]
|
||||
[
|
||||
jobId,
|
||||
result.productsFound,
|
||||
result.productsUpserted,
|
||||
JSON.stringify(metadata),
|
||||
]
|
||||
);
|
||||
console.log(`[JobQueue] Job ${jobId} completed`);
|
||||
}
|
||||
|
||||
@@ -24,12 +24,8 @@ import {
|
||||
} from '../types';
|
||||
import { downloadProductImage, imageExists } from '../../utils/image-storage';
|
||||
|
||||
// Explicit column list for dispensaries table (avoids SELECT * issues with schema differences)
|
||||
const DISPENSARY_COLUMNS = `
|
||||
id, name, slug, city, state, zip, address, latitude, longitude,
|
||||
menu_type, menu_url, platform_dispensary_id, website,
|
||||
provider_detection_data, created_at, updated_at
|
||||
`;
|
||||
// Use shared dispensary columns (handles optional columns like provider_detection_data)
|
||||
import { DISPENSARY_COLUMNS } from '../db/dispensary-columns';
|
||||
|
||||
// ============================================================
|
||||
// BATCH PROCESSING CONFIGURATION
|
||||
@@ -648,10 +644,15 @@ async function updateDispensaryCrawlStats(
|
||||
}
|
||||
|
||||
/**
|
||||
* Mark products as missing from feed
|
||||
* Mark products as missing from feed (visibility-loss detection)
|
||||
* Creates a snapshot with isPresentInFeed=false and stockStatus='missing_from_feed'
|
||||
* for products that were NOT in the UNION of Mode A and Mode B product lists
|
||||
*
|
||||
* Bella (Product Sync) visibility tracking:
|
||||
* - Sets visibility_lost=TRUE and visibility_lost_at=NOW() for disappearing products
|
||||
* - Records visibility event in snapshot metadata JSONB
|
||||
* - NEVER deletes products, just marks them as visibility-lost
|
||||
*
|
||||
* IMPORTANT: Uses UNION of both modes to avoid false positives
|
||||
* If the union is empty (possible outage), we skip marking to avoid data corruption
|
||||
*/
|
||||
@@ -660,25 +661,28 @@ async function markMissingProducts(
|
||||
platformDispensaryId: string,
|
||||
modeAProductIds: Set<string>,
|
||||
modeBProductIds: Set<string>,
|
||||
pricingType: 'rec' | 'med'
|
||||
): Promise<number> {
|
||||
pricingType: 'rec' | 'med',
|
||||
workerName: string = 'Bella'
|
||||
): Promise<{ markedMissing: number; newlyLost: number }> {
|
||||
// Build UNION of Mode A + Mode B product IDs
|
||||
const unionProductIds = new Set<string>([...Array.from(modeAProductIds), ...Array.from(modeBProductIds)]);
|
||||
|
||||
// OUTAGE DETECTION: If union is empty, something went wrong - don't mark anything as missing
|
||||
if (unionProductIds.size === 0) {
|
||||
console.warn('[ProductCrawler] OUTAGE DETECTED: Both Mode A and Mode B returned 0 products. Skipping missing product marking.');
|
||||
return 0;
|
||||
console.warn(`[${workerName} - Product Sync] OUTAGE DETECTED: Both Mode A and Mode B returned 0 products. Skipping visibility-loss marking.`);
|
||||
return { markedMissing: 0, newlyLost: 0 };
|
||||
}
|
||||
|
||||
// Get all existing products for this dispensary that were not in the UNION
|
||||
// Also check if they were already marked as visibility_lost to track new losses
|
||||
const { rows: missingProducts } = await query<{
|
||||
id: number;
|
||||
external_product_id: string;
|
||||
name: string;
|
||||
visibility_lost: boolean;
|
||||
}>(
|
||||
`
|
||||
SELECT id, external_product_id, name
|
||||
SELECT id, external_product_id, name, COALESCE(visibility_lost, FALSE) as visibility_lost
|
||||
FROM dutchie_products
|
||||
WHERE dispensary_id = $1
|
||||
AND external_product_id NOT IN (SELECT unnest($2::text[]))
|
||||
@@ -687,59 +691,141 @@ async function markMissingProducts(
|
||||
);
|
||||
|
||||
if (missingProducts.length === 0) {
|
||||
return 0;
|
||||
return { markedMissing: 0, newlyLost: 0 };
|
||||
}
|
||||
|
||||
console.log(`[ProductCrawler] Marking ${missingProducts.length} products as missing from feed (union of ${modeAProductIds.size} Mode A + ${modeBProductIds.size} Mode B = ${unionProductIds.size} unique)...`);
|
||||
// Separate newly lost products from already-lost products
|
||||
const newlyLostProducts = missingProducts.filter(p => !p.visibility_lost);
|
||||
const alreadyLostProducts = missingProducts.filter(p => p.visibility_lost);
|
||||
|
||||
console.log(`[${workerName} - Product Sync] Visibility check: ${missingProducts.length} products missing (${newlyLostProducts.length} newly lost, ${alreadyLostProducts.length} already lost)`);
|
||||
|
||||
const crawledAt = new Date();
|
||||
|
||||
// Build all missing snapshots first (per CLAUDE.md Rule #15 - batch writes)
|
||||
const missingSnapshots: Partial<DutchieProductSnapshot>[] = missingProducts.map(product => ({
|
||||
dutchieProductId: product.id,
|
||||
dispensaryId,
|
||||
platformDispensaryId,
|
||||
externalProductId: product.external_product_id,
|
||||
pricingType,
|
||||
crawlMode: 'mode_a' as CrawlMode, // Use mode_a for missing snapshots (convention)
|
||||
status: undefined,
|
||||
featured: false,
|
||||
special: false,
|
||||
medicalOnly: false,
|
||||
recOnly: false,
|
||||
isPresentInFeed: false,
|
||||
stockStatus: 'missing_from_feed' as StockStatus,
|
||||
totalQuantityAvailable: undefined, // null = unknown, not 0
|
||||
manualInventory: false,
|
||||
isBelowThreshold: false,
|
||||
isBelowKioskThreshold: false,
|
||||
options: [],
|
||||
rawPayload: { _missingFromFeed: true, lastKnownName: product.name },
|
||||
crawledAt,
|
||||
}));
|
||||
// Build all missing snapshots with visibility_events metadata
|
||||
const missingSnapshots: Partial<DutchieProductSnapshot>[] = missingProducts.map(product => {
|
||||
const isNewlyLost = !product.visibility_lost;
|
||||
return {
|
||||
dutchieProductId: product.id,
|
||||
dispensaryId,
|
||||
platformDispensaryId,
|
||||
externalProductId: product.external_product_id,
|
||||
pricingType,
|
||||
crawlMode: 'mode_a' as CrawlMode,
|
||||
status: undefined,
|
||||
featured: false,
|
||||
special: false,
|
||||
medicalOnly: false,
|
||||
recOnly: false,
|
||||
isPresentInFeed: false,
|
||||
stockStatus: 'missing_from_feed' as StockStatus,
|
||||
totalQuantityAvailable: undefined,
|
||||
manualInventory: false,
|
||||
isBelowThreshold: false,
|
||||
isBelowKioskThreshold: false,
|
||||
options: [],
|
||||
rawPayload: {
|
||||
_missingFromFeed: true,
|
||||
lastKnownName: product.name,
|
||||
visibility_events: isNewlyLost ? [{
|
||||
event_type: 'visibility_lost',
|
||||
timestamp: crawledAt.toISOString(),
|
||||
worker_name: workerName,
|
||||
}] : [],
|
||||
},
|
||||
crawledAt,
|
||||
};
|
||||
});
|
||||
|
||||
// Batch insert missing snapshots
|
||||
const snapshotsInserted = await batchInsertSnapshots(missingSnapshots);
|
||||
|
||||
// Batch update product stock status in chunks
|
||||
// Batch update product visibility status in chunks
|
||||
const productIds = missingProducts.map(p => p.id);
|
||||
const productChunks = chunkArray(productIds, BATCH_CHUNK_SIZE);
|
||||
|
||||
console.log(`[ProductCrawler] Updating ${productIds.length} product statuses in ${productChunks.length} chunks...`);
|
||||
console.log(`[${workerName} - Product Sync] Updating ${productIds.length} product visibility in ${productChunks.length} chunks...`);
|
||||
|
||||
for (const chunk of productChunks) {
|
||||
// Update all products: set stock_status to missing
|
||||
// Only set visibility_lost_at for NEWLY lost products (not already lost)
|
||||
await query(
|
||||
`
|
||||
UPDATE dutchie_products
|
||||
SET stock_status = 'missing_from_feed', total_quantity_available = NULL, updated_at = NOW()
|
||||
SET
|
||||
stock_status = 'missing_from_feed',
|
||||
total_quantity_available = NULL,
|
||||
visibility_lost = TRUE,
|
||||
visibility_lost_at = CASE
|
||||
WHEN visibility_lost IS NULL OR visibility_lost = FALSE THEN NOW()
|
||||
ELSE visibility_lost_at -- Keep existing timestamp for already-lost products
|
||||
END,
|
||||
updated_at = NOW()
|
||||
WHERE id = ANY($1::int[])
|
||||
`,
|
||||
[chunk]
|
||||
);
|
||||
}
|
||||
|
||||
console.log(`[ProductCrawler] Marked ${snapshotsInserted} products as missing from feed`);
|
||||
return snapshotsInserted;
|
||||
console.log(`[${workerName} - Product Sync] Marked ${snapshotsInserted} products as missing, ${newlyLostProducts.length} newly visibility-lost`);
|
||||
return { markedMissing: snapshotsInserted, newlyLost: newlyLostProducts.length };
|
||||
}
|
||||
|
||||
/**
|
||||
* Restore visibility for products that reappeared in the feed
|
||||
* Called when products that were previously visibility_lost=TRUE are now found in the feed
|
||||
*
|
||||
* Bella (Product Sync) visibility tracking:
|
||||
* - Sets visibility_lost=FALSE and visibility_restored_at=NOW()
|
||||
* - Logs the restoration event
|
||||
*/
|
||||
async function restoreVisibilityForProducts(
|
||||
dispensaryId: number,
|
||||
productIds: Set<string>,
|
||||
workerName: string = 'Bella'
|
||||
): Promise<number> {
|
||||
if (productIds.size === 0) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
// Find products that were visibility_lost and are now in the feed
|
||||
const { rows: restoredProducts } = await query<{ id: number; external_product_id: string }>(
|
||||
`
|
||||
SELECT id, external_product_id
|
||||
FROM dutchie_products
|
||||
WHERE dispensary_id = $1
|
||||
AND visibility_lost = TRUE
|
||||
AND external_product_id = ANY($2::text[])
|
||||
`,
|
||||
[dispensaryId, Array.from(productIds)]
|
||||
);
|
||||
|
||||
if (restoredProducts.length === 0) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
console.log(`[${workerName} - Product Sync] Restoring visibility for ${restoredProducts.length} products that reappeared`);
|
||||
|
||||
// Batch update restored products
|
||||
const restoredIds = restoredProducts.map(p => p.id);
|
||||
const chunks = chunkArray(restoredIds, BATCH_CHUNK_SIZE);
|
||||
|
||||
for (const chunk of chunks) {
|
||||
await query(
|
||||
`
|
||||
UPDATE dutchie_products
|
||||
SET
|
||||
visibility_lost = FALSE,
|
||||
visibility_restored_at = NOW(),
|
||||
updated_at = NOW()
|
||||
WHERE id = ANY($1::int[])
|
||||
`,
|
||||
[chunk]
|
||||
);
|
||||
}
|
||||
|
||||
console.log(`[${workerName} - Product Sync] Restored visibility for ${restoredProducts.length} products`);
|
||||
return restoredProducts.length;
|
||||
}
|
||||
|
||||
// ============================================================
|
||||
@@ -756,9 +842,12 @@ export interface CrawlResult {
|
||||
modeAProducts?: number;
|
||||
modeBProducts?: number;
|
||||
missingProductsMarked?: number;
|
||||
visibilityLostCount?: number; // Products newly marked as visibility_lost
|
||||
visibilityRestoredCount?: number; // Products restored from visibility_lost
|
||||
imagesDownloaded?: number;
|
||||
imageErrors?: number;
|
||||
errorMessage?: string;
|
||||
httpStatus?: number; // HTTP status code for error classification
|
||||
durationMs: number;
|
||||
}
|
||||
|
||||
@@ -1005,21 +1094,38 @@ export async function crawlDispensaryProducts(
|
||||
}
|
||||
}
|
||||
|
||||
// Build union of all product IDs found in both modes
|
||||
const allFoundProductIds = new Set<string>([
|
||||
...Array.from(modeAProductIds),
|
||||
...Array.from(modeBProductIds),
|
||||
]);
|
||||
|
||||
// VISIBILITY RESTORATION: Check if any previously-lost products have reappeared
|
||||
const visibilityRestored = await restoreVisibilityForProducts(
|
||||
dispensary.id,
|
||||
allFoundProductIds,
|
||||
'Bella'
|
||||
);
|
||||
|
||||
// Mark products as missing using UNION of Mode A + Mode B
|
||||
// The function handles outage detection (empty union = skip marking)
|
||||
missingMarked = await markMissingProducts(
|
||||
// Now also tracks newly lost products vs already-lost products
|
||||
const missingResult = await markMissingProducts(
|
||||
dispensary.id,
|
||||
dispensary.platformDispensaryId,
|
||||
modeAProductIds,
|
||||
modeBProductIds,
|
||||
pricingType
|
||||
pricingType,
|
||||
'Bella'
|
||||
);
|
||||
missingMarked = missingResult.markedMissing;
|
||||
const newlyLostCount = missingResult.newlyLost;
|
||||
totalSnapshots += missingMarked;
|
||||
|
||||
// Update dispensary stats
|
||||
await updateDispensaryCrawlStats(dispensary.id, totalUpserted);
|
||||
|
||||
console.log(`[ProductCrawler] Completed: ${totalUpserted} products, ${totalSnapshots} snapshots, ${missingMarked} marked missing, ${totalImagesDownloaded} images downloaded`);
|
||||
console.log(`[Bella - Product Sync] Completed: ${totalUpserted} products, ${totalSnapshots} snapshots, ${missingMarked} missing, ${newlyLostCount} newly lost, ${visibilityRestored} restored, ${totalImagesDownloaded} images`);
|
||||
|
||||
const totalProductsFound = modeAProducts + modeBProducts;
|
||||
return {
|
||||
@@ -1032,6 +1138,8 @@ export async function crawlDispensaryProducts(
|
||||
modeAProducts,
|
||||
modeBProducts,
|
||||
missingProductsMarked: missingMarked,
|
||||
visibilityLostCount: newlyLostCount,
|
||||
visibilityRestoredCount: visibilityRestored,
|
||||
imagesDownloaded: totalImagesDownloaded,
|
||||
imageErrors: totalImageErrors,
|
||||
durationMs: Date.now() - startTime,
|
||||
|
||||
@@ -3,6 +3,8 @@
|
||||
*
|
||||
* Polls the job queue and processes crawl jobs.
|
||||
* Each worker instance runs independently, claiming jobs atomically.
|
||||
*
|
||||
* Phase 1: Enhanced with self-healing logic, error taxonomy, and retry management.
|
||||
*/
|
||||
|
||||
import {
|
||||
@@ -20,13 +22,36 @@ import { crawlDispensaryProducts } from './product-crawler';
|
||||
import { mapDbRowToDispensary } from './discovery';
|
||||
import { query } from '../db/connection';
|
||||
|
||||
// Explicit column list for dispensaries table (avoids SELECT * issues with schema differences)
|
||||
// NOTE: failed_at is included for worker compatibility checks
|
||||
const DISPENSARY_COLUMNS = `
|
||||
id, name, slug, city, state, zip, address, latitude, longitude,
|
||||
menu_type, menu_url, platform_dispensary_id, website,
|
||||
provider_detection_data, created_at, updated_at, failed_at
|
||||
`;
|
||||
// Phase 1: Error taxonomy and retry management
|
||||
import {
|
||||
CrawlErrorCode,
|
||||
CrawlErrorCodeType,
|
||||
classifyError,
|
||||
isRetryable,
|
||||
shouldRotateProxy,
|
||||
shouldRotateUserAgent,
|
||||
createSuccessResult,
|
||||
createFailureResult,
|
||||
CrawlResult,
|
||||
} from './error-taxonomy';
|
||||
import {
|
||||
RetryManager,
|
||||
RetryDecision,
|
||||
calculateNextCrawlAt,
|
||||
determineCrawlStatus,
|
||||
shouldAttemptRecovery,
|
||||
sleep,
|
||||
} from './retry-manager';
|
||||
import {
|
||||
CrawlRotator,
|
||||
userAgentRotator,
|
||||
updateDispensaryRotation,
|
||||
} from './proxy-rotator';
|
||||
import { DEFAULT_CONFIG, validateStoreConfig, isCrawlable } from './store-validator';
|
||||
|
||||
// Use shared dispensary columns (handles optional columns like provider_detection_data)
|
||||
// NOTE: Using WITH_FAILED variant for worker compatibility checks
|
||||
import { DISPENSARY_COLUMNS_WITH_FAILED as DISPENSARY_COLUMNS } from '../db/dispensary-columns';
|
||||
|
||||
// ============================================================
|
||||
// WORKER CONFIG
|
||||
@@ -236,66 +261,245 @@ async function processJob(job: QueuedJob): Promise<void> {
|
||||
}
|
||||
}
|
||||
|
||||
// Maximum consecutive failures before flagging a dispensary
|
||||
const MAX_CONSECUTIVE_FAILURES = 3;
|
||||
// Thresholds for crawl status transitions
|
||||
const DEGRADED_THRESHOLD = 3; // Mark as degraded after 3 consecutive failures
|
||||
const FAILED_THRESHOLD = 10; // Mark as failed after 10 consecutive failures
|
||||
|
||||
// For backwards compatibility
|
||||
const MAX_CONSECUTIVE_FAILURES = FAILED_THRESHOLD;
|
||||
|
||||
/**
|
||||
* Record a successful crawl - resets failure counter
|
||||
* Record a successful crawl - resets failure counter and restores active status
|
||||
*/
|
||||
async function recordCrawlSuccess(dispensaryId: number): Promise<void> {
|
||||
async function recordCrawlSuccess(
|
||||
dispensaryId: number,
|
||||
result: CrawlResult
|
||||
): Promise<void> {
|
||||
// Calculate next crawl time (use store's frequency or default)
|
||||
const { rows: storeRows } = await query<any>(
|
||||
`SELECT crawl_frequency_minutes FROM dispensaries WHERE id = $1`,
|
||||
[dispensaryId]
|
||||
);
|
||||
const frequencyMinutes = storeRows[0]?.crawl_frequency_minutes || DEFAULT_CONFIG.crawlFrequencyMinutes;
|
||||
const nextCrawlAt = calculateNextCrawlAt(0, frequencyMinutes);
|
||||
|
||||
// Reset failure state and schedule next crawl
|
||||
await query(
|
||||
`UPDATE dispensaries
|
||||
SET consecutive_failures = 0,
|
||||
crawl_status = 'active',
|
||||
backoff_multiplier = 1.0,
|
||||
last_crawl_at = NOW(),
|
||||
last_success_at = NOW(),
|
||||
last_error_code = NULL,
|
||||
next_crawl_at = $2,
|
||||
total_attempts = COALESCE(total_attempts, 0) + 1,
|
||||
total_successes = COALESCE(total_successes, 0) + 1,
|
||||
updated_at = NOW()
|
||||
WHERE id = $1`,
|
||||
[dispensaryId]
|
||||
[dispensaryId, nextCrawlAt]
|
||||
);
|
||||
|
||||
// Log to crawl_attempts table for analytics
|
||||
await logCrawlAttempt(dispensaryId, result);
|
||||
|
||||
console.log(`[Worker] Dispensary ${dispensaryId} crawl success. Next crawl at ${nextCrawlAt.toISOString()}`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Record a crawl failure - increments counter and may flag dispensary
|
||||
* Returns true if dispensary was flagged as failed
|
||||
* Record a crawl failure with self-healing logic
|
||||
* - Rotates proxy/UA based on error type
|
||||
* - Transitions through: active -> degraded -> failed
|
||||
* - Calculates backoff for next attempt
|
||||
*/
|
||||
async function recordCrawlFailure(dispensaryId: number, errorMessage: string): Promise<boolean> {
|
||||
// Increment failure counter
|
||||
const { rows } = await query<any>(
|
||||
`UPDATE dispensaries
|
||||
SET consecutive_failures = consecutive_failures + 1,
|
||||
last_failure_at = NOW(),
|
||||
last_failure_reason = $2,
|
||||
updated_at = NOW()
|
||||
WHERE id = $1
|
||||
RETURNING consecutive_failures`,
|
||||
[dispensaryId, errorMessage]
|
||||
async function recordCrawlFailure(
|
||||
dispensaryId: number,
|
||||
errorMessage: string,
|
||||
errorCode?: CrawlErrorCodeType,
|
||||
httpStatus?: number,
|
||||
context?: {
|
||||
proxyUsed?: string;
|
||||
userAgentUsed?: string;
|
||||
attemptNumber?: number;
|
||||
}
|
||||
): Promise<{ wasFlagged: boolean; newStatus: string; shouldRotateProxy: boolean; shouldRotateUA: boolean }> {
|
||||
// Classify the error if not provided
|
||||
const code = errorCode || classifyError(errorMessage, httpStatus);
|
||||
|
||||
// Get current state
|
||||
const { rows: storeRows } = await query<any>(
|
||||
`SELECT
|
||||
consecutive_failures,
|
||||
crawl_status,
|
||||
backoff_multiplier,
|
||||
crawl_frequency_minutes,
|
||||
current_proxy_id,
|
||||
current_user_agent
|
||||
FROM dispensaries WHERE id = $1`,
|
||||
[dispensaryId]
|
||||
);
|
||||
|
||||
const failures = rows[0]?.consecutive_failures || 0;
|
||||
|
||||
// If we've hit the threshold, flag the dispensary as failed
|
||||
if (failures >= MAX_CONSECUTIVE_FAILURES) {
|
||||
await query(
|
||||
`UPDATE dispensaries
|
||||
SET failed_at = NOW(),
|
||||
menu_type = NULL,
|
||||
platform_dispensary_id = NULL,
|
||||
failure_notes = $2,
|
||||
updated_at = NOW()
|
||||
WHERE id = $1`,
|
||||
[dispensaryId, `Auto-flagged after ${failures} consecutive failures. Last error: ${errorMessage}`]
|
||||
);
|
||||
console.log(`[Worker] Dispensary ${dispensaryId} flagged as FAILED after ${failures} consecutive failures`);
|
||||
return true;
|
||||
if (storeRows.length === 0) {
|
||||
return { wasFlagged: false, newStatus: 'unknown', shouldRotateProxy: false, shouldRotateUA: false };
|
||||
}
|
||||
|
||||
console.log(`[Worker] Dispensary ${dispensaryId} failure recorded (${failures}/${MAX_CONSECUTIVE_FAILURES})`);
|
||||
return false;
|
||||
const store = storeRows[0];
|
||||
const currentFailures = (store.consecutive_failures || 0) + 1;
|
||||
const frequencyMinutes = store.crawl_frequency_minutes || DEFAULT_CONFIG.crawlFrequencyMinutes;
|
||||
|
||||
// Determine if we should rotate proxy/UA based on error type
|
||||
const rotateProxy = shouldRotateProxy(code);
|
||||
const rotateUA = shouldRotateUserAgent(code);
|
||||
|
||||
// Get new proxy/UA if rotation is needed
|
||||
let newProxyId = store.current_proxy_id;
|
||||
let newUserAgent = store.current_user_agent;
|
||||
|
||||
if (rotateUA) {
|
||||
newUserAgent = userAgentRotator.getNext();
|
||||
console.log(`[Worker] Rotating user agent for dispensary ${dispensaryId} after ${code}`);
|
||||
}
|
||||
|
||||
// Determine new crawl status
|
||||
const newStatus = determineCrawlStatus(currentFailures, {
|
||||
degraded: DEGRADED_THRESHOLD,
|
||||
failed: FAILED_THRESHOLD,
|
||||
});
|
||||
|
||||
// Calculate backoff multiplier and next crawl time
|
||||
const newBackoffMultiplier = Math.min(
|
||||
(store.backoff_multiplier || 1.0) * 1.5,
|
||||
4.0 // Max 4x backoff
|
||||
);
|
||||
const nextCrawlAt = calculateNextCrawlAt(currentFailures, frequencyMinutes);
|
||||
|
||||
// Update dispensary with new failure state
|
||||
if (newStatus === 'failed') {
|
||||
// Mark as failed - won't be crawled again until manual intervention
|
||||
await query(
|
||||
`UPDATE dispensaries
|
||||
SET consecutive_failures = $2,
|
||||
crawl_status = $3,
|
||||
backoff_multiplier = $4,
|
||||
last_failure_at = NOW(),
|
||||
last_error_code = $5,
|
||||
failed_at = NOW(),
|
||||
failure_notes = $6,
|
||||
next_crawl_at = NULL,
|
||||
current_proxy_id = $7,
|
||||
current_user_agent = $8,
|
||||
total_attempts = COALESCE(total_attempts, 0) + 1,
|
||||
updated_at = NOW()
|
||||
WHERE id = $1`,
|
||||
[
|
||||
dispensaryId,
|
||||
currentFailures,
|
||||
newStatus,
|
||||
newBackoffMultiplier,
|
||||
code,
|
||||
`Auto-flagged after ${currentFailures} consecutive failures. Last error: ${errorMessage}`,
|
||||
newProxyId,
|
||||
newUserAgent,
|
||||
]
|
||||
);
|
||||
console.log(`[Worker] Dispensary ${dispensaryId} marked as FAILED after ${currentFailures} failures (${code})`);
|
||||
} else {
|
||||
// Update failure count but keep crawling (active or degraded)
|
||||
await query(
|
||||
`UPDATE dispensaries
|
||||
SET consecutive_failures = $2,
|
||||
crawl_status = $3,
|
||||
backoff_multiplier = $4,
|
||||
last_failure_at = NOW(),
|
||||
last_error_code = $5,
|
||||
next_crawl_at = $6,
|
||||
current_proxy_id = $7,
|
||||
current_user_agent = $8,
|
||||
total_attempts = COALESCE(total_attempts, 0) + 1,
|
||||
updated_at = NOW()
|
||||
WHERE id = $1`,
|
||||
[
|
||||
dispensaryId,
|
||||
currentFailures,
|
||||
newStatus,
|
||||
newBackoffMultiplier,
|
||||
code,
|
||||
nextCrawlAt,
|
||||
newProxyId,
|
||||
newUserAgent,
|
||||
]
|
||||
);
|
||||
|
||||
if (newStatus === 'degraded') {
|
||||
console.log(`[Worker] Dispensary ${dispensaryId} marked as DEGRADED (${currentFailures}/${FAILED_THRESHOLD} failures). Next crawl: ${nextCrawlAt.toISOString()}`);
|
||||
} else {
|
||||
console.log(`[Worker] Dispensary ${dispensaryId} failure recorded (${currentFailures}/${DEGRADED_THRESHOLD}). Next crawl: ${nextCrawlAt.toISOString()}`);
|
||||
}
|
||||
}
|
||||
|
||||
// Log to crawl_attempts table
|
||||
const result = createFailureResult(
|
||||
dispensaryId,
|
||||
new Date(),
|
||||
errorMessage,
|
||||
httpStatus,
|
||||
context
|
||||
);
|
||||
await logCrawlAttempt(dispensaryId, result);
|
||||
|
||||
return {
|
||||
wasFlagged: newStatus === 'failed',
|
||||
newStatus,
|
||||
shouldRotateProxy: rotateProxy,
|
||||
shouldRotateUA: rotateUA,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Log a crawl attempt to the crawl_attempts table for analytics
|
||||
*/
|
||||
async function logCrawlAttempt(
|
||||
dispensaryId: number,
|
||||
result: CrawlResult
|
||||
): Promise<void> {
|
||||
try {
|
||||
await query(
|
||||
`INSERT INTO crawl_attempts (
|
||||
dispensary_id, started_at, finished_at, duration_ms,
|
||||
error_code, error_message, http_status,
|
||||
attempt_number, proxy_used, user_agent_used,
|
||||
products_found, products_upserted, snapshots_created,
|
||||
created_at
|
||||
) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, NOW())`,
|
||||
[
|
||||
dispensaryId,
|
||||
result.startedAt,
|
||||
result.finishedAt,
|
||||
result.durationMs,
|
||||
result.errorCode,
|
||||
result.errorMessage || null,
|
||||
result.httpStatus || null,
|
||||
result.attemptNumber,
|
||||
result.proxyUsed || null,
|
||||
result.userAgentUsed || null,
|
||||
result.productsFound || 0,
|
||||
result.productsUpserted || 0,
|
||||
result.snapshotsCreated || 0,
|
||||
]
|
||||
);
|
||||
} catch (error) {
|
||||
// Don't fail the job if logging fails
|
||||
console.error(`[Worker] Failed to log crawl attempt for dispensary ${dispensaryId}:`, error);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Process a product crawl job for a single dispensary
|
||||
*/
|
||||
async function processProductCrawlJob(job: QueuedJob): Promise<void> {
|
||||
const startedAt = new Date();
|
||||
const userAgent = userAgentRotator.getCurrent();
|
||||
|
||||
if (!job.dispensaryId) {
|
||||
throw new Error('Product crawl job requires dispensary_id');
|
||||
}
|
||||
@@ -311,17 +515,35 @@ async function processProductCrawlJob(job: QueuedJob): Promise<void> {
|
||||
}
|
||||
|
||||
const dispensary = mapDbRowToDispensary(rows[0]);
|
||||
const rawDispensary = rows[0];
|
||||
|
||||
// Check if dispensary is already flagged as failed
|
||||
if (rows[0].failed_at) {
|
||||
if (rawDispensary.failed_at) {
|
||||
console.log(`[Worker] Skipping dispensary ${job.dispensaryId} - already flagged as failed`);
|
||||
await completeJob(job.id, { productsFound: 0, productsUpserted: 0 });
|
||||
return;
|
||||
}
|
||||
|
||||
// Check crawl status - skip if paused or failed
|
||||
if (rawDispensary.crawl_status === 'paused' || rawDispensary.crawl_status === 'failed') {
|
||||
console.log(`[Worker] Skipping dispensary ${job.dispensaryId} - crawl_status is ${rawDispensary.crawl_status}`);
|
||||
await completeJob(job.id, { productsFound: 0, productsUpserted: 0 });
|
||||
return;
|
||||
}
|
||||
|
||||
if (!dispensary.platformDispensaryId) {
|
||||
// Record failure and potentially flag
|
||||
await recordCrawlFailure(job.dispensaryId, 'Missing platform_dispensary_id');
|
||||
// Record failure with error taxonomy
|
||||
const { wasFlagged } = await recordCrawlFailure(
|
||||
job.dispensaryId,
|
||||
'Missing platform_dispensary_id',
|
||||
CrawlErrorCode.MISSING_PLATFORM_ID,
|
||||
undefined,
|
||||
{ userAgentUsed: userAgent, attemptNumber: job.retryCount + 1 }
|
||||
);
|
||||
if (wasFlagged) {
|
||||
await completeJob(job.id, { productsFound: 0, productsUpserted: 0 });
|
||||
return;
|
||||
}
|
||||
throw new Error(`Dispensary ${job.dispensaryId} has no platform_dispensary_id`);
|
||||
}
|
||||
|
||||
@@ -346,28 +568,67 @@ async function processProductCrawlJob(job: QueuedJob): Promise<void> {
|
||||
});
|
||||
|
||||
if (result.success) {
|
||||
// Success! Reset failure counter
|
||||
await recordCrawlSuccess(job.dispensaryId);
|
||||
// Success! Create result and record
|
||||
const crawlResult = createSuccessResult(
|
||||
job.dispensaryId,
|
||||
startedAt,
|
||||
{
|
||||
productsFound: result.productsFetched,
|
||||
productsUpserted: result.productsUpserted,
|
||||
snapshotsCreated: result.snapshotsCreated,
|
||||
},
|
||||
{
|
||||
attemptNumber: job.retryCount + 1,
|
||||
userAgentUsed: userAgent,
|
||||
}
|
||||
);
|
||||
await recordCrawlSuccess(job.dispensaryId, crawlResult);
|
||||
await completeJob(job.id, {
|
||||
productsFound: result.productsFetched,
|
||||
productsUpserted: result.productsUpserted,
|
||||
snapshotsCreated: result.snapshotsCreated,
|
||||
// Visibility tracking stats for dashboard
|
||||
visibilityLostCount: result.visibilityLostCount || 0,
|
||||
visibilityRestoredCount: result.visibilityRestoredCount || 0,
|
||||
});
|
||||
} else {
|
||||
// Crawl returned failure - record it
|
||||
const wasFlagged = await recordCrawlFailure(job.dispensaryId, result.errorMessage || 'Crawl failed');
|
||||
// Crawl returned failure - classify error and record
|
||||
const errorCode = classifyError(result.errorMessage || 'Crawl failed', result.httpStatus);
|
||||
const { wasFlagged } = await recordCrawlFailure(
|
||||
job.dispensaryId,
|
||||
result.errorMessage || 'Crawl failed',
|
||||
errorCode,
|
||||
result.httpStatus,
|
||||
{ userAgentUsed: userAgent, attemptNumber: job.retryCount + 1 }
|
||||
);
|
||||
|
||||
if (wasFlagged) {
|
||||
// Don't throw - the dispensary is now flagged, job is "complete"
|
||||
// Dispensary is now flagged - complete the job
|
||||
await completeJob(job.id, { productsFound: 0, productsUpserted: 0 });
|
||||
} else if (!isRetryable(errorCode)) {
|
||||
// Non-retryable error - complete as failed
|
||||
await completeJob(job.id, { productsFound: 0, productsUpserted: 0 });
|
||||
} else {
|
||||
// Retryable error - let job queue handle retry
|
||||
throw new Error(result.errorMessage || 'Crawl failed');
|
||||
}
|
||||
}
|
||||
} catch (error: any) {
|
||||
// Record the failure
|
||||
const wasFlagged = await recordCrawlFailure(job.dispensaryId, error.message);
|
||||
// Record the failure with error taxonomy
|
||||
const errorCode = classifyError(error.message);
|
||||
const { wasFlagged } = await recordCrawlFailure(
|
||||
job.dispensaryId,
|
||||
error.message,
|
||||
errorCode,
|
||||
undefined,
|
||||
{ userAgentUsed: userAgent, attemptNumber: job.retryCount + 1 }
|
||||
);
|
||||
|
||||
if (wasFlagged) {
|
||||
// Dispensary is now flagged - complete the job rather than fail it
|
||||
// Dispensary is now flagged - complete the job
|
||||
await completeJob(job.id, { productsFound: 0, productsUpserted: 0 });
|
||||
} else if (!isRetryable(errorCode)) {
|
||||
// Non-retryable error - complete as failed
|
||||
await completeJob(job.id, { productsFound: 0, productsUpserted: 0 });
|
||||
} else {
|
||||
throw error;
|
||||
|
||||
339
backend/src/multi-state/__tests__/state-query-service.test.ts
Normal file
339
backend/src/multi-state/__tests__/state-query-service.test.ts
Normal file
@@ -0,0 +1,339 @@
|
||||
/**
|
||||
* StateQueryService Unit Tests
|
||||
* Phase 4: Multi-State Expansion
|
||||
*/
|
||||
|
||||
import { StateQueryService } from '../state-query-service';
|
||||
|
||||
// Mock the pool
|
||||
const mockQuery = jest.fn();
|
||||
const mockPool = {
|
||||
query: mockQuery,
|
||||
} as any;
|
||||
|
||||
describe('StateQueryService', () => {
|
||||
let service: StateQueryService;
|
||||
|
||||
beforeEach(() => {
|
||||
jest.clearAllMocks();
|
||||
service = new StateQueryService(mockPool);
|
||||
});
|
||||
|
||||
describe('listStates', () => {
|
||||
it('should return all states ordered by name', async () => {
|
||||
mockQuery.mockResolvedValueOnce({
|
||||
rows: [
|
||||
{ code: 'AZ', name: 'Arizona' },
|
||||
{ code: 'CA', name: 'California' },
|
||||
{ code: 'NV', name: 'Nevada' },
|
||||
],
|
||||
});
|
||||
|
||||
const states = await service.listStates();
|
||||
|
||||
expect(states).toHaveLength(3);
|
||||
expect(states[0].code).toBe('AZ');
|
||||
expect(mockQuery).toHaveBeenCalledWith(expect.stringContaining('FROM states'));
|
||||
});
|
||||
});
|
||||
|
||||
describe('listActiveStates', () => {
|
||||
it('should return only states with dispensary data', async () => {
|
||||
mockQuery.mockResolvedValueOnce({
|
||||
rows: [
|
||||
{ code: 'AZ', name: 'Arizona' },
|
||||
],
|
||||
});
|
||||
|
||||
const states = await service.listActiveStates();
|
||||
|
||||
expect(states).toHaveLength(1);
|
||||
expect(states[0].code).toBe('AZ');
|
||||
expect(mockQuery).toHaveBeenCalledWith(
|
||||
expect.stringContaining('JOIN dispensaries')
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('getStateSummary', () => {
|
||||
it('should return null for unknown state', async () => {
|
||||
// Materialized view query returns empty
|
||||
mockQuery.mockResolvedValueOnce({ rows: [] });
|
||||
|
||||
const summary = await service.getStateSummary('XX');
|
||||
|
||||
expect(summary).toBeNull();
|
||||
});
|
||||
|
||||
it('should return full summary for valid state', async () => {
|
||||
// Materialized view query
|
||||
mockQuery.mockResolvedValueOnce({
|
||||
rows: [{
|
||||
state: 'AZ',
|
||||
stateName: 'Arizona',
|
||||
storeCount: 100,
|
||||
dutchieStores: 80,
|
||||
activeStores: 75,
|
||||
totalProducts: 5000,
|
||||
inStockProducts: 4500,
|
||||
onSpecialProducts: 200,
|
||||
uniqueBrands: 150,
|
||||
uniqueCategories: 10,
|
||||
avgPriceRec: 45.50,
|
||||
minPriceRec: 10.00,
|
||||
maxPriceRec: 200.00,
|
||||
refreshedAt: new Date(),
|
||||
}],
|
||||
});
|
||||
|
||||
// Crawl stats query
|
||||
mockQuery.mockResolvedValueOnce({
|
||||
rows: [{
|
||||
recent_crawls: '50',
|
||||
failed_crawls: '2',
|
||||
last_crawl_at: new Date(),
|
||||
}],
|
||||
});
|
||||
|
||||
// Top brands
|
||||
mockQuery.mockResolvedValueOnce({
|
||||
rows: [
|
||||
{ brandId: 1, brandName: 'Brand 1', storeCount: 50, productCount: 100 },
|
||||
],
|
||||
});
|
||||
|
||||
// Top categories
|
||||
mockQuery.mockResolvedValueOnce({
|
||||
rows: [
|
||||
{ category: 'Flower', productCount: 1000, storeCount: 80 },
|
||||
],
|
||||
});
|
||||
|
||||
const summary = await service.getStateSummary('AZ');
|
||||
|
||||
expect(summary).not.toBeNull();
|
||||
expect(summary!.state).toBe('AZ');
|
||||
expect(summary!.storeCount).toBe(100);
|
||||
expect(summary!.totalProducts).toBe(5000);
|
||||
expect(summary!.recentCrawls).toBe(50);
|
||||
expect(summary!.topBrands).toHaveLength(1);
|
||||
expect(summary!.topCategories).toHaveLength(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe('getBrandsByState', () => {
|
||||
it('should return brands for a state with pagination', async () => {
|
||||
mockQuery.mockResolvedValueOnce({
|
||||
rows: [
|
||||
{
|
||||
brandId: 1,
|
||||
brandName: 'Brand A',
|
||||
brandSlug: 'brand-a',
|
||||
storeCount: 50,
|
||||
productCount: 200,
|
||||
avgPrice: 45.00,
|
||||
firstSeenInState: new Date(),
|
||||
lastSeenInState: new Date(),
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
const brands = await service.getBrandsByState('AZ', { limit: 10, offset: 0 });
|
||||
|
||||
expect(brands).toHaveLength(1);
|
||||
expect(brands[0].brandName).toBe('Brand A');
|
||||
expect(mockQuery).toHaveBeenCalledWith(
|
||||
expect.stringContaining('FROM v_brand_state_presence'),
|
||||
['AZ', 10, 0]
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('getBrandStatePenetration', () => {
|
||||
it('should return penetration data for a brand', async () => {
|
||||
mockQuery.mockResolvedValueOnce({
|
||||
rows: [
|
||||
{
|
||||
state: 'AZ',
|
||||
stateName: 'Arizona',
|
||||
totalStores: 100,
|
||||
storesWithBrand: 50,
|
||||
penetrationPct: 50.00,
|
||||
productCount: 200,
|
||||
avgPrice: 45.00,
|
||||
},
|
||||
{
|
||||
state: 'CA',
|
||||
stateName: 'California',
|
||||
totalStores: 200,
|
||||
storesWithBrand: 60,
|
||||
penetrationPct: 30.00,
|
||||
productCount: 300,
|
||||
avgPrice: 55.00,
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
const penetration = await service.getBrandStatePenetration(123);
|
||||
|
||||
expect(penetration).toHaveLength(2);
|
||||
expect(penetration[0].state).toBe('AZ');
|
||||
expect(penetration[0].penetrationPct).toBe(50.00);
|
||||
expect(penetration[1].state).toBe('CA');
|
||||
});
|
||||
});
|
||||
|
||||
describe('compareBrandAcrossStates', () => {
|
||||
it('should compare brand across specified states', async () => {
|
||||
// Brand lookup
|
||||
mockQuery.mockResolvedValueOnce({
|
||||
rows: [{ id: 1, name: 'Test Brand' }],
|
||||
});
|
||||
|
||||
// Penetration data
|
||||
mockQuery.mockResolvedValueOnce({
|
||||
rows: [
|
||||
{ state: 'AZ', stateName: 'Arizona', penetrationPct: 50, totalStores: 100, storesWithBrand: 50, productCount: 200, avgPrice: 45 },
|
||||
{ state: 'CA', stateName: 'California', penetrationPct: 30, totalStores: 200, storesWithBrand: 60, productCount: 300, avgPrice: 55 },
|
||||
],
|
||||
});
|
||||
|
||||
// National stats
|
||||
mockQuery.mockResolvedValueOnce({
|
||||
rows: [{ total_stores: '300', stores_with_brand: '110', avg_price: 50 }],
|
||||
});
|
||||
|
||||
const comparison = await service.compareBrandAcrossStates(1, ['AZ', 'CA']);
|
||||
|
||||
expect(comparison.brandName).toBe('Test Brand');
|
||||
expect(comparison.states).toHaveLength(2);
|
||||
expect(comparison.bestPerformingState).toBe('AZ');
|
||||
expect(comparison.worstPerformingState).toBe('CA');
|
||||
});
|
||||
|
||||
it('should throw error for unknown brand', async () => {
|
||||
mockQuery.mockResolvedValueOnce({ rows: [] });
|
||||
|
||||
await expect(service.compareBrandAcrossStates(999, ['AZ'])).rejects.toThrow('Brand 999 not found');
|
||||
});
|
||||
});
|
||||
|
||||
describe('getCategoriesByState', () => {
|
||||
it('should return categories for a state', async () => {
|
||||
mockQuery.mockResolvedValueOnce({
|
||||
rows: [
|
||||
{ category: 'Flower', productCount: 1000, storeCount: 80, avgPrice: 35, inStockCount: 900, onSpecialCount: 50 },
|
||||
{ category: 'Edibles', productCount: 500, storeCount: 60, avgPrice: 25, inStockCount: 450, onSpecialCount: 30 },
|
||||
],
|
||||
});
|
||||
|
||||
const categories = await service.getCategoriesByState('AZ');
|
||||
|
||||
expect(categories).toHaveLength(2);
|
||||
expect(categories[0].category).toBe('Flower');
|
||||
expect(categories[0].productCount).toBe(1000);
|
||||
});
|
||||
});
|
||||
|
||||
describe('getStoresByState', () => {
|
||||
it('should return stores with metrics', async () => {
|
||||
mockQuery.mockResolvedValueOnce({
|
||||
rows: [
|
||||
{
|
||||
dispensaryId: 1,
|
||||
dispensaryName: 'Test Store',
|
||||
dispensarySlug: 'test-store',
|
||||
state: 'AZ',
|
||||
city: 'Phoenix',
|
||||
menuType: 'dutchie',
|
||||
crawlStatus: 'active',
|
||||
lastCrawlAt: new Date(),
|
||||
productCount: 200,
|
||||
inStockCount: 180,
|
||||
brandCount: 50,
|
||||
avgPrice: 45,
|
||||
specialCount: 10,
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
const stores = await service.getStoresByState('AZ', { limit: 50 });
|
||||
|
||||
expect(stores).toHaveLength(1);
|
||||
expect(stores[0].dispensaryName).toBe('Test Store');
|
||||
expect(stores[0].productCount).toBe(200);
|
||||
});
|
||||
});
|
||||
|
||||
describe('getNationalSummary', () => {
|
||||
it('should return national aggregate metrics', async () => {
|
||||
// State metrics
|
||||
mockQuery.mockResolvedValueOnce({
|
||||
rows: [
|
||||
{ state: 'AZ', stateName: 'Arizona', storeCount: 100, totalProducts: 5000 },
|
||||
],
|
||||
});
|
||||
|
||||
// National counts
|
||||
mockQuery.mockResolvedValueOnce({
|
||||
rows: [{
|
||||
total_states: '17',
|
||||
active_states: '5',
|
||||
total_stores: '500',
|
||||
total_products: '25000',
|
||||
total_brands: '300',
|
||||
avg_price_national: 45.00,
|
||||
}],
|
||||
});
|
||||
|
||||
const summary = await service.getNationalSummary();
|
||||
|
||||
expect(summary.totalStates).toBe(17);
|
||||
expect(summary.activeStates).toBe(5);
|
||||
expect(summary.totalStores).toBe(500);
|
||||
expect(summary.totalProducts).toBe(25000);
|
||||
expect(summary.avgPriceNational).toBe(45.00);
|
||||
});
|
||||
});
|
||||
|
||||
describe('getStateHeatmapData', () => {
|
||||
it('should return heatmap data for stores metric', async () => {
|
||||
mockQuery.mockResolvedValueOnce({
|
||||
rows: [
|
||||
{ state: 'AZ', stateName: 'Arizona', value: 100, label: 'stores' },
|
||||
{ state: 'CA', stateName: 'California', value: 200, label: 'stores' },
|
||||
],
|
||||
});
|
||||
|
||||
const heatmap = await service.getStateHeatmapData('stores');
|
||||
|
||||
expect(heatmap).toHaveLength(2);
|
||||
expect(heatmap[0].state).toBe('AZ');
|
||||
expect(heatmap[0].value).toBe(100);
|
||||
});
|
||||
|
||||
it('should require brandId for penetration metric', async () => {
|
||||
await expect(service.getStateHeatmapData('penetration')).rejects.toThrow(
|
||||
'brandId required for penetration heatmap'
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('isValidState', () => {
|
||||
it('should return true for valid state code', async () => {
|
||||
mockQuery.mockResolvedValueOnce({ rows: [{ code: 'AZ' }] });
|
||||
|
||||
const isValid = await service.isValidState('AZ');
|
||||
|
||||
expect(isValid).toBe(true);
|
||||
});
|
||||
|
||||
it('should return false for invalid state code', async () => {
|
||||
mockQuery.mockResolvedValueOnce({ rows: [] });
|
||||
|
||||
const isValid = await service.isValidState('XX');
|
||||
|
||||
expect(isValid).toBe(false);
|
||||
});
|
||||
});
|
||||
});
|
||||
15
backend/src/multi-state/index.ts
Normal file
15
backend/src/multi-state/index.ts
Normal file
@@ -0,0 +1,15 @@
|
||||
/**
|
||||
* Multi-State Module
|
||||
*
|
||||
* Central export for multi-state queries and analytics.
|
||||
* Phase 4: Multi-State Expansion
|
||||
*/
|
||||
|
||||
// Types
|
||||
export * from './types';
|
||||
|
||||
// Query Service
|
||||
export { StateQueryService } from './state-query-service';
|
||||
|
||||
// Routes
|
||||
export { createMultiStateRoutes } from './routes';
|
||||
451
backend/src/multi-state/routes.ts
Normal file
451
backend/src/multi-state/routes.ts
Normal file
@@ -0,0 +1,451 @@
|
||||
/**
|
||||
* Multi-State API Routes
|
||||
*
|
||||
* Endpoints for multi-state queries, analytics, and comparisons.
|
||||
* Phase 4: Multi-State Expansion
|
||||
*/
|
||||
|
||||
import { Router, Request, Response } from 'express';
|
||||
import { Pool } from 'pg';
|
||||
import { StateQueryService } from './state-query-service';
|
||||
import { StateQueryOptions, CrossStateQueryOptions } from './types';
|
||||
|
||||
export function createMultiStateRoutes(pool: Pool): Router {
|
||||
const router = Router();
|
||||
const stateService = new StateQueryService(pool);
|
||||
|
||||
// =========================================================================
|
||||
// State List Endpoints
|
||||
// =========================================================================
|
||||
|
||||
/**
|
||||
* GET /api/states
|
||||
* List all states (both configured and active)
|
||||
*/
|
||||
router.get('/states', async (req: Request, res: Response) => {
|
||||
try {
|
||||
const activeOnly = req.query.active === 'true';
|
||||
const states = activeOnly
|
||||
? await stateService.listActiveStates()
|
||||
: await stateService.listStates();
|
||||
|
||||
res.json({
|
||||
success: true,
|
||||
data: {
|
||||
states,
|
||||
count: states.length,
|
||||
},
|
||||
});
|
||||
} catch (error: any) {
|
||||
console.error('[MultiState] Error listing states:', error);
|
||||
res.status(500).json({ success: false, error: error.message });
|
||||
}
|
||||
});
|
||||
|
||||
// =========================================================================
|
||||
// State Summary Endpoints
|
||||
// =========================================================================
|
||||
|
||||
/**
|
||||
* GET /api/state/:state/summary
|
||||
* Get detailed summary for a specific state
|
||||
*/
|
||||
router.get('/state/:state/summary', async (req: Request, res: Response) => {
|
||||
try {
|
||||
const { state } = req.params;
|
||||
|
||||
// Validate state code
|
||||
const isValid = await stateService.isValidState(state.toUpperCase());
|
||||
if (!isValid) {
|
||||
return res.status(404).json({
|
||||
success: false,
|
||||
error: `Unknown state: ${state}`,
|
||||
});
|
||||
}
|
||||
|
||||
const summary = await stateService.getStateSummary(state.toUpperCase());
|
||||
|
||||
if (!summary) {
|
||||
return res.status(404).json({
|
||||
success: false,
|
||||
error: `No data for state: ${state}`,
|
||||
});
|
||||
}
|
||||
|
||||
res.json({
|
||||
success: true,
|
||||
data: summary,
|
||||
});
|
||||
} catch (error: any) {
|
||||
console.error(`[MultiState] Error getting state summary:`, error);
|
||||
res.status(500).json({ success: false, error: error.message });
|
||||
}
|
||||
});
|
||||
|
||||
/**
|
||||
* GET /api/state/:state/brands
|
||||
* Get brands in a specific state
|
||||
*/
|
||||
router.get('/state/:state/brands', async (req: Request, res: Response) => {
|
||||
try {
|
||||
const { state } = req.params;
|
||||
const options: StateQueryOptions = {
|
||||
limit: parseInt(req.query.limit as string) || 50,
|
||||
offset: parseInt(req.query.offset as string) || 0,
|
||||
sortBy: (req.query.sortBy as string) || 'productCount',
|
||||
sortDir: (req.query.sortDir as 'asc' | 'desc') || 'desc',
|
||||
};
|
||||
|
||||
const brands = await stateService.getBrandsByState(state.toUpperCase(), options);
|
||||
|
||||
res.json({
|
||||
success: true,
|
||||
data: {
|
||||
state: state.toUpperCase(),
|
||||
brands,
|
||||
count: brands.length,
|
||||
pagination: {
|
||||
limit: options.limit,
|
||||
offset: options.offset,
|
||||
},
|
||||
},
|
||||
});
|
||||
} catch (error: any) {
|
||||
console.error(`[MultiState] Error getting state brands:`, error);
|
||||
res.status(500).json({ success: false, error: error.message });
|
||||
}
|
||||
});
|
||||
|
||||
/**
|
||||
* GET /api/state/:state/categories
|
||||
* Get categories in a specific state
|
||||
*/
|
||||
router.get('/state/:state/categories', async (req: Request, res: Response) => {
|
||||
try {
|
||||
const { state } = req.params;
|
||||
const options: StateQueryOptions = {
|
||||
limit: parseInt(req.query.limit as string) || 50,
|
||||
offset: parseInt(req.query.offset as string) || 0,
|
||||
sortBy: (req.query.sortBy as string) || 'productCount',
|
||||
sortDir: (req.query.sortDir as 'asc' | 'desc') || 'desc',
|
||||
};
|
||||
|
||||
const categories = await stateService.getCategoriesByState(state.toUpperCase(), options);
|
||||
|
||||
res.json({
|
||||
success: true,
|
||||
data: {
|
||||
state: state.toUpperCase(),
|
||||
categories,
|
||||
count: categories.length,
|
||||
pagination: {
|
||||
limit: options.limit,
|
||||
offset: options.offset,
|
||||
},
|
||||
},
|
||||
});
|
||||
} catch (error: any) {
|
||||
console.error(`[MultiState] Error getting state categories:`, error);
|
||||
res.status(500).json({ success: false, error: error.message });
|
||||
}
|
||||
});
|
||||
|
||||
/**
|
||||
* GET /api/state/:state/stores
|
||||
* Get stores in a specific state
|
||||
*/
|
||||
router.get('/state/:state/stores', async (req: Request, res: Response) => {
|
||||
try {
|
||||
const { state } = req.params;
|
||||
const options: StateQueryOptions = {
|
||||
limit: parseInt(req.query.limit as string) || 100,
|
||||
offset: parseInt(req.query.offset as string) || 0,
|
||||
sortBy: (req.query.sortBy as string) || 'productCount',
|
||||
sortDir: (req.query.sortDir as 'asc' | 'desc') || 'desc',
|
||||
includeInactive: req.query.includeInactive === 'true',
|
||||
};
|
||||
|
||||
const stores = await stateService.getStoresByState(state.toUpperCase(), options);
|
||||
|
||||
res.json({
|
||||
success: true,
|
||||
data: {
|
||||
state: state.toUpperCase(),
|
||||
stores,
|
||||
count: stores.length,
|
||||
pagination: {
|
||||
limit: options.limit,
|
||||
offset: options.offset,
|
||||
},
|
||||
},
|
||||
});
|
||||
} catch (error: any) {
|
||||
console.error(`[MultiState] Error getting state stores:`, error);
|
||||
res.status(500).json({ success: false, error: error.message });
|
||||
}
|
||||
});
|
||||
|
||||
/**
|
||||
* GET /api/state/:state/analytics/prices
|
||||
* Get price distribution for a state
|
||||
*/
|
||||
router.get('/state/:state/analytics/prices', async (req: Request, res: Response) => {
|
||||
try {
|
||||
const { state } = req.params;
|
||||
const options = {
|
||||
category: req.query.category as string | undefined,
|
||||
brandId: req.query.brandId ? parseInt(req.query.brandId as string) : undefined,
|
||||
};
|
||||
|
||||
const priceData = await stateService.getStorePriceDistribution(
|
||||
state.toUpperCase(),
|
||||
options
|
||||
);
|
||||
|
||||
res.json({
|
||||
success: true,
|
||||
data: {
|
||||
state: state.toUpperCase(),
|
||||
priceDistribution: priceData,
|
||||
},
|
||||
});
|
||||
} catch (error: any) {
|
||||
console.error(`[MultiState] Error getting price analytics:`, error);
|
||||
res.status(500).json({ success: false, error: error.message });
|
||||
}
|
||||
});
|
||||
|
||||
// =========================================================================
|
||||
// National Analytics Endpoints
|
||||
// =========================================================================
|
||||
|
||||
/**
|
||||
* GET /api/analytics/national/summary
|
||||
* Get national summary across all states
|
||||
*/
|
||||
router.get('/analytics/national/summary', async (req: Request, res: Response) => {
|
||||
try {
|
||||
const summary = await stateService.getNationalSummary();
|
||||
|
||||
res.json({
|
||||
success: true,
|
||||
data: summary,
|
||||
});
|
||||
} catch (error: any) {
|
||||
console.error(`[MultiState] Error getting national summary:`, error);
|
||||
res.status(500).json({ success: false, error: error.message });
|
||||
}
|
||||
});
|
||||
|
||||
/**
|
||||
* GET /api/analytics/national/prices
|
||||
* Get national price comparison across all states
|
||||
*/
|
||||
router.get('/analytics/national/prices', async (req: Request, res: Response) => {
|
||||
try {
|
||||
const options = {
|
||||
category: req.query.category as string | undefined,
|
||||
brandId: req.query.brandId ? parseInt(req.query.brandId as string) : undefined,
|
||||
};
|
||||
|
||||
const priceData = await stateService.getNationalPriceComparison(options);
|
||||
|
||||
res.json({
|
||||
success: true,
|
||||
data: {
|
||||
priceComparison: priceData,
|
||||
filters: options,
|
||||
},
|
||||
});
|
||||
} catch (error: any) {
|
||||
console.error(`[MultiState] Error getting national prices:`, error);
|
||||
res.status(500).json({ success: false, error: error.message });
|
||||
}
|
||||
});
|
||||
|
||||
/**
|
||||
* GET /api/analytics/national/heatmap
|
||||
* GET /api/national/heatmap (alias)
|
||||
* Get state heatmap data for various metrics
|
||||
*/
|
||||
const heatmapHandler = async (req: Request, res: Response) => {
|
||||
try {
|
||||
const metric = (req.query.metric as 'stores' | 'products' | 'brands' | 'avgPrice' | 'penetration') || 'stores';
|
||||
const brandId = req.query.brandId ? parseInt(req.query.brandId as string) : undefined;
|
||||
const category = req.query.category as string | undefined;
|
||||
|
||||
const heatmapData = await stateService.getStateHeatmapData(metric, { brandId, category });
|
||||
|
||||
res.json({
|
||||
success: true,
|
||||
data: {
|
||||
metric,
|
||||
heatmap: heatmapData,
|
||||
},
|
||||
});
|
||||
} catch (error: any) {
|
||||
console.error(`[MultiState] Error getting heatmap data:`, error);
|
||||
res.status(500).json({ success: false, error: error.message });
|
||||
}
|
||||
};
|
||||
|
||||
// Register heatmap on both paths for compatibility
|
||||
router.get('/analytics/national/heatmap', heatmapHandler);
|
||||
router.get('/national/heatmap', heatmapHandler);
|
||||
|
||||
/**
|
||||
* GET /api/analytics/national/metrics
|
||||
* Get all state metrics for dashboard
|
||||
*/
|
||||
router.get('/analytics/national/metrics', async (req: Request, res: Response) => {
|
||||
try {
|
||||
const metrics = await stateService.getAllStateMetrics();
|
||||
|
||||
res.json({
|
||||
success: true,
|
||||
data: {
|
||||
stateMetrics: metrics,
|
||||
count: metrics.length,
|
||||
},
|
||||
});
|
||||
} catch (error: any) {
|
||||
console.error(`[MultiState] Error getting state metrics:`, error);
|
||||
res.status(500).json({ success: false, error: error.message });
|
||||
}
|
||||
});
|
||||
|
||||
// =========================================================================
|
||||
// Cross-State Comparison Endpoints
|
||||
// =========================================================================
|
||||
|
||||
/**
|
||||
* GET /api/analytics/compare/brand/:brandId
|
||||
* Compare a brand across multiple states
|
||||
*/
|
||||
router.get('/analytics/compare/brand/:brandId', async (req: Request, res: Response) => {
|
||||
try {
|
||||
const brandId = parseInt(req.params.brandId);
|
||||
const statesParam = req.query.states as string;
|
||||
|
||||
// Parse states - either comma-separated or get all active states
|
||||
let states: string[];
|
||||
if (statesParam) {
|
||||
states = statesParam.split(',').map(s => s.trim().toUpperCase());
|
||||
} else {
|
||||
const activeStates = await stateService.listActiveStates();
|
||||
states = activeStates.map(s => s.code);
|
||||
}
|
||||
|
||||
const comparison = await stateService.compareBrandAcrossStates(brandId, states);
|
||||
|
||||
res.json({
|
||||
success: true,
|
||||
data: comparison,
|
||||
});
|
||||
} catch (error: any) {
|
||||
console.error(`[MultiState] Error comparing brand across states:`, error);
|
||||
res.status(500).json({ success: false, error: error.message });
|
||||
}
|
||||
});
|
||||
|
||||
/**
|
||||
* GET /api/analytics/compare/category/:category
|
||||
* Compare a category across multiple states
|
||||
*/
|
||||
router.get('/analytics/compare/category/:category', async (req: Request, res: Response) => {
|
||||
try {
|
||||
const { category } = req.params;
|
||||
const statesParam = req.query.states as string;
|
||||
|
||||
// Parse states - either comma-separated or get all active states
|
||||
let states: string[];
|
||||
if (statesParam) {
|
||||
states = statesParam.split(',').map(s => s.trim().toUpperCase());
|
||||
} else {
|
||||
const activeStates = await stateService.listActiveStates();
|
||||
states = activeStates.map(s => s.code);
|
||||
}
|
||||
|
||||
const comparison = await stateService.compareCategoryAcrossStates(category, states);
|
||||
|
||||
res.json({
|
||||
success: true,
|
||||
data: comparison,
|
||||
});
|
||||
} catch (error: any) {
|
||||
console.error(`[MultiState] Error comparing category across states:`, error);
|
||||
res.status(500).json({ success: false, error: error.message });
|
||||
}
|
||||
});
|
||||
|
||||
/**
|
||||
* GET /api/analytics/brand/:brandId/penetration
|
||||
* Get brand penetration across all states
|
||||
*/
|
||||
router.get('/analytics/brand/:brandId/penetration', async (req: Request, res: Response) => {
|
||||
try {
|
||||
const brandId = parseInt(req.params.brandId);
|
||||
|
||||
const penetration = await stateService.getBrandStatePenetration(brandId);
|
||||
|
||||
res.json({
|
||||
success: true,
|
||||
data: {
|
||||
brandId,
|
||||
statePenetration: penetration,
|
||||
},
|
||||
});
|
||||
} catch (error: any) {
|
||||
console.error(`[MultiState] Error getting brand penetration:`, error);
|
||||
res.status(500).json({ success: false, error: error.message });
|
||||
}
|
||||
});
|
||||
|
||||
/**
|
||||
* GET /api/analytics/brand/:brandId/trend
|
||||
* Get national penetration trend for a brand
|
||||
*/
|
||||
router.get('/analytics/brand/:brandId/trend', async (req: Request, res: Response) => {
|
||||
try {
|
||||
const brandId = parseInt(req.params.brandId);
|
||||
const days = parseInt(req.query.days as string) || 30;
|
||||
|
||||
const trend = await stateService.getNationalPenetrationTrend(brandId, { days });
|
||||
|
||||
res.json({
|
||||
success: true,
|
||||
data: trend,
|
||||
});
|
||||
} catch (error: any) {
|
||||
console.error(`[MultiState] Error getting brand trend:`, error);
|
||||
res.status(500).json({ success: false, error: error.message });
|
||||
}
|
||||
});
|
||||
|
||||
// =========================================================================
|
||||
// Admin Endpoints
|
||||
// =========================================================================
|
||||
|
||||
/**
|
||||
* POST /api/admin/states/refresh-metrics
|
||||
* Manually refresh materialized views
|
||||
*/
|
||||
router.post('/admin/states/refresh-metrics', async (req: Request, res: Response) => {
|
||||
try {
|
||||
const startTime = Date.now();
|
||||
await stateService.refreshMetrics();
|
||||
const duration = Date.now() - startTime;
|
||||
|
||||
res.json({
|
||||
success: true,
|
||||
message: 'State metrics refreshed successfully',
|
||||
durationMs: duration,
|
||||
});
|
||||
} catch (error: any) {
|
||||
console.error(`[MultiState] Error refreshing metrics:`, error);
|
||||
res.status(500).json({ success: false, error: error.message });
|
||||
}
|
||||
});
|
||||
|
||||
return router;
|
||||
}
|
||||
643
backend/src/multi-state/state-query-service.ts
Normal file
643
backend/src/multi-state/state-query-service.ts
Normal file
@@ -0,0 +1,643 @@
|
||||
/**
|
||||
* StateQueryService
|
||||
*
|
||||
* Core service for multi-state queries and analytics.
|
||||
* Phase 4: Multi-State Expansion
|
||||
*/
|
||||
|
||||
import { Pool } from 'pg';
|
||||
import {
|
||||
State,
|
||||
StateMetrics,
|
||||
StateSummary,
|
||||
BrandInState,
|
||||
BrandStatePenetration,
|
||||
BrandCrossStateComparison,
|
||||
CategoryInState,
|
||||
CategoryStateDist,
|
||||
CategoryCrossStateComparison,
|
||||
StoreInState,
|
||||
StatePriceDistribution,
|
||||
NationalSummary,
|
||||
NationalPenetrationTrend,
|
||||
StateHeatmapData,
|
||||
StateQueryOptions,
|
||||
CrossStateQueryOptions,
|
||||
} from './types';
|
||||
|
||||
export class StateQueryService {
|
||||
constructor(private pool: Pool) {}
|
||||
|
||||
// =========================================================================
|
||||
// State List & Basic Queries
|
||||
// =========================================================================
|
||||
|
||||
/**
|
||||
* Get all available states
|
||||
*/
|
||||
async listStates(): Promise<State[]> {
|
||||
const result = await this.pool.query(`
|
||||
SELECT code, name
|
||||
FROM states
|
||||
ORDER BY name
|
||||
`);
|
||||
return result.rows;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get states that have dispensary data
|
||||
*/
|
||||
async listActiveStates(): Promise<State[]> {
|
||||
const result = await this.pool.query(`
|
||||
SELECT DISTINCT s.code, s.name
|
||||
FROM states s
|
||||
JOIN dispensaries d ON d.state = s.code
|
||||
WHERE d.menu_type IS NOT NULL
|
||||
ORDER BY s.name
|
||||
`);
|
||||
return result.rows;
|
||||
}
|
||||
|
||||
// =========================================================================
|
||||
// State Summary & Metrics
|
||||
// =========================================================================
|
||||
|
||||
/**
|
||||
* Get summary metrics for a single state
|
||||
*/
|
||||
async getStateSummary(state: string): Promise<StateSummary | null> {
|
||||
// Get base metrics from materialized view
|
||||
const metricsResult = await this.pool.query(`
|
||||
SELECT
|
||||
state,
|
||||
state_name AS "stateName",
|
||||
dispensary_count AS "storeCount",
|
||||
dispensary_count AS "dutchieStores",
|
||||
dispensary_count AS "activeStores",
|
||||
total_products AS "totalProducts",
|
||||
in_stock_products AS "inStockProducts",
|
||||
out_of_stock_products AS "outOfStockProducts",
|
||||
unique_brands AS "uniqueBrands",
|
||||
unique_categories AS "uniqueCategories",
|
||||
avg_price_rec AS "avgPriceRec",
|
||||
min_price_rec AS "minPriceRec",
|
||||
max_price_rec AS "maxPriceRec",
|
||||
refreshed_at AS "refreshedAt"
|
||||
FROM mv_state_metrics
|
||||
WHERE state = $1
|
||||
`, [state]);
|
||||
|
||||
if (metricsResult.rows.length === 0) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const metrics = metricsResult.rows[0];
|
||||
|
||||
// Get crawl stats
|
||||
const crawlResult = await this.pool.query(`
|
||||
SELECT
|
||||
COUNT(*) FILTER (WHERE cr.status = 'success' AND cr.started_at > NOW() - INTERVAL '24 hours') AS recent_crawls,
|
||||
COUNT(*) FILTER (WHERE cr.status = 'failed' AND cr.started_at > NOW() - INTERVAL '24 hours') AS failed_crawls,
|
||||
MAX(cr.finished_at) AS last_crawl_at
|
||||
FROM crawl_runs cr
|
||||
JOIN dispensaries d ON cr.dispensary_id = d.id
|
||||
WHERE d.state = $1
|
||||
`, [state]);
|
||||
|
||||
// Get top brands
|
||||
const topBrands = await this.getBrandsByState(state, { limit: 5 });
|
||||
|
||||
// Get top categories
|
||||
const topCategories = await this.getCategoriesByState(state, { limit: 5 });
|
||||
|
||||
return {
|
||||
...metrics,
|
||||
recentCrawls: parseInt(crawlResult.rows[0]?.recent_crawls || '0'),
|
||||
failedCrawls: parseInt(crawlResult.rows[0]?.failed_crawls || '0'),
|
||||
lastCrawlAt: crawlResult.rows[0]?.last_crawl_at || null,
|
||||
topBrands,
|
||||
topCategories,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Get metrics for all states
|
||||
*/
|
||||
async getAllStateMetrics(): Promise<StateMetrics[]> {
|
||||
const result = await this.pool.query(`
|
||||
SELECT
|
||||
state,
|
||||
state_name AS "stateName",
|
||||
dispensary_count AS "storeCount",
|
||||
dispensary_count AS "dutchieStores",
|
||||
dispensary_count AS "activeStores",
|
||||
total_products AS "totalProducts",
|
||||
in_stock_products AS "inStockProducts",
|
||||
out_of_stock_products AS "outOfStockProducts",
|
||||
unique_brands AS "uniqueBrands",
|
||||
unique_categories AS "uniqueCategories",
|
||||
avg_price_rec AS "avgPriceRec",
|
||||
min_price_rec AS "minPriceRec",
|
||||
max_price_rec AS "maxPriceRec",
|
||||
refreshed_at AS "refreshedAt"
|
||||
FROM mv_state_metrics
|
||||
ORDER BY dispensary_count DESC
|
||||
`);
|
||||
return result.rows;
|
||||
}
|
||||
|
||||
// =========================================================================
|
||||
// Brand Queries
|
||||
// =========================================================================
|
||||
|
||||
/**
|
||||
* Get brands present in a specific state
|
||||
*/
|
||||
async getBrandsByState(state: string, options: StateQueryOptions = {}): Promise<BrandInState[]> {
|
||||
const { limit = 50, offset = 0, sortBy = 'productCount', sortDir = 'desc' } = options;
|
||||
|
||||
const sortColumn = {
|
||||
productCount: 'product_count',
|
||||
storeCount: 'store_count',
|
||||
avgPrice: 'avg_price',
|
||||
name: 'brand_name',
|
||||
}[sortBy] || 'product_count';
|
||||
|
||||
const result = await this.pool.query(`
|
||||
SELECT
|
||||
brand_id AS "brandId",
|
||||
brand_name AS "brandName",
|
||||
brand_slug AS "brandSlug",
|
||||
store_count AS "storeCount",
|
||||
product_count AS "productCount",
|
||||
avg_price AS "avgPrice",
|
||||
first_seen_in_state AS "firstSeenInState",
|
||||
last_seen_in_state AS "lastSeenInState"
|
||||
FROM v_brand_state_presence
|
||||
WHERE state = $1
|
||||
ORDER BY ${sortColumn} ${sortDir === 'asc' ? 'ASC' : 'DESC'}
|
||||
LIMIT $2 OFFSET $3
|
||||
`, [state, limit, offset]);
|
||||
|
||||
return result.rows;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get brand penetration across all states
|
||||
*/
|
||||
async getBrandStatePenetration(brandId: number): Promise<BrandStatePenetration[]> {
|
||||
const result = await this.pool.query(`
|
||||
SELECT
|
||||
state,
|
||||
state_name AS "stateName",
|
||||
total_stores AS "totalStores",
|
||||
stores_with_brand AS "storesWithBrand",
|
||||
penetration_pct AS "penetrationPct",
|
||||
product_count AS "productCount",
|
||||
avg_price AS "avgPrice"
|
||||
FROM fn_brand_state_penetration($1)
|
||||
`, [brandId]);
|
||||
|
||||
return result.rows;
|
||||
}
|
||||
|
||||
/**
|
||||
* Compare a brand across multiple states
|
||||
*/
|
||||
async compareBrandAcrossStates(
|
||||
brandId: number,
|
||||
states: string[]
|
||||
): Promise<BrandCrossStateComparison> {
|
||||
// Get brand info
|
||||
const brandResult = await this.pool.query(`
|
||||
SELECT id, name FROM brands WHERE id = $1
|
||||
`, [brandId]);
|
||||
|
||||
if (brandResult.rows.length === 0) {
|
||||
throw new Error(`Brand ${brandId} not found`);
|
||||
}
|
||||
|
||||
const brand = brandResult.rows[0];
|
||||
|
||||
// Get penetration for specified states
|
||||
const allPenetration = await this.getBrandStatePenetration(brandId);
|
||||
const filteredStates = allPenetration.filter(p => states.includes(p.state));
|
||||
|
||||
// Calculate national metrics
|
||||
const nationalResult = await this.pool.query(`
|
||||
SELECT
|
||||
COUNT(DISTINCT d.id) AS total_stores,
|
||||
COUNT(DISTINCT CASE WHEN sp.brand_id = $1 THEN d.id END) AS stores_with_brand,
|
||||
AVG(sp.price_rec) FILTER (WHERE sp.brand_id = $1) AS avg_price
|
||||
FROM dispensaries d
|
||||
LEFT JOIN store_products sp ON d.id = sp.dispensary_id
|
||||
WHERE d.state IS NOT NULL
|
||||
`, [brandId]);
|
||||
|
||||
const nationalData = nationalResult.rows[0];
|
||||
const nationalPenetration = nationalData.total_stores > 0
|
||||
? (nationalData.stores_with_brand / nationalData.total_stores) * 100
|
||||
: 0;
|
||||
|
||||
// Find best/worst states
|
||||
const sortedByPenetration = [...filteredStates].sort(
|
||||
(a, b) => b.penetrationPct - a.penetrationPct
|
||||
);
|
||||
|
||||
return {
|
||||
brandId,
|
||||
brandName: brand.name,
|
||||
states: filteredStates,
|
||||
nationalPenetration: Math.round(nationalPenetration * 100) / 100,
|
||||
nationalAvgPrice: nationalData.avg_price
|
||||
? Math.round(nationalData.avg_price * 100) / 100
|
||||
: null,
|
||||
bestPerformingState: sortedByPenetration[0]?.state || null,
|
||||
worstPerformingState: sortedByPenetration[sortedByPenetration.length - 1]?.state || null,
|
||||
};
|
||||
}
|
||||
|
||||
// =========================================================================
|
||||
// Category Queries
|
||||
// =========================================================================
|
||||
|
||||
/**
|
||||
* Get categories in a specific state
|
||||
*/
|
||||
async getCategoriesByState(state: string, options: StateQueryOptions = {}): Promise<CategoryInState[]> {
|
||||
const { limit = 50, offset = 0, sortBy = 'productCount', sortDir = 'desc' } = options;
|
||||
|
||||
const sortColumn = {
|
||||
productCount: 'product_count',
|
||||
storeCount: 'store_count',
|
||||
avgPrice: 'avg_price',
|
||||
category: 'category',
|
||||
}[sortBy] || 'product_count';
|
||||
|
||||
const result = await this.pool.query(`
|
||||
SELECT
|
||||
category,
|
||||
product_count AS "productCount",
|
||||
store_count AS "storeCount",
|
||||
avg_price AS "avgPrice",
|
||||
in_stock_count AS "inStockCount",
|
||||
on_special_count AS "onSpecialCount"
|
||||
FROM v_category_state_distribution
|
||||
WHERE state = $1
|
||||
ORDER BY ${sortColumn} ${sortDir === 'asc' ? 'ASC' : 'DESC'}
|
||||
LIMIT $2 OFFSET $3
|
||||
`, [state, limit, offset]);
|
||||
|
||||
return result.rows;
|
||||
}
|
||||
|
||||
/**
|
||||
* Compare a category across multiple states
|
||||
*/
|
||||
async compareCategoryAcrossStates(
|
||||
category: string,
|
||||
states: string[]
|
||||
): Promise<CategoryCrossStateComparison> {
|
||||
const result = await this.pool.query(`
|
||||
SELECT
|
||||
v.state,
|
||||
s.name AS "stateName",
|
||||
v.category,
|
||||
v.product_count AS "productCount",
|
||||
v.store_count AS "storeCount",
|
||||
v.avg_price AS "avgPrice",
|
||||
ROUND(v.product_count::NUMERIC / SUM(v.product_count) OVER () * 100, 2) AS "marketShare"
|
||||
FROM v_category_state_distribution v
|
||||
JOIN states s ON v.state = s.code
|
||||
WHERE v.category = $1
|
||||
AND v.state = ANY($2)
|
||||
ORDER BY v.product_count DESC
|
||||
`, [category, states]);
|
||||
|
||||
// Get national totals
|
||||
const nationalResult = await this.pool.query(`
|
||||
SELECT
|
||||
COUNT(DISTINCT sp.id) AS product_count,
|
||||
AVG(sp.price_rec) AS avg_price
|
||||
FROM store_products sp
|
||||
WHERE sp.category_raw = $1
|
||||
`, [category]);
|
||||
|
||||
const national = nationalResult.rows[0];
|
||||
|
||||
// Find dominant state
|
||||
const dominantState = result.rows.length > 0 ? result.rows[0].state : null;
|
||||
|
||||
return {
|
||||
category,
|
||||
states: result.rows,
|
||||
nationalProductCount: parseInt(national.product_count || '0'),
|
||||
nationalAvgPrice: national.avg_price
|
||||
? Math.round(national.avg_price * 100) / 100
|
||||
: null,
|
||||
dominantState,
|
||||
};
|
||||
}
|
||||
|
||||
// =========================================================================
|
||||
// Store Queries
|
||||
// =========================================================================
|
||||
|
||||
/**
|
||||
* Get stores in a specific state
|
||||
*/
|
||||
async getStoresByState(state: string, options: StateQueryOptions = {}): Promise<StoreInState[]> {
|
||||
const { limit = 100, offset = 0, includeInactive = false, sortBy = 'productCount', sortDir = 'desc' } = options;
|
||||
|
||||
const sortColumn = {
|
||||
productCount: 'product_count',
|
||||
brandCount: 'brand_count',
|
||||
avgPrice: 'avg_price',
|
||||
name: 'dispensary_name',
|
||||
city: 'city',
|
||||
lastCrawl: 'last_crawl_at',
|
||||
}[sortBy] || 'product_count';
|
||||
|
||||
let whereClause = 'WHERE state = $1';
|
||||
if (!includeInactive) {
|
||||
whereClause += ` AND crawl_status != 'disabled'`;
|
||||
}
|
||||
|
||||
const result = await this.pool.query(`
|
||||
SELECT
|
||||
dispensary_id AS "dispensaryId",
|
||||
dispensary_name AS "dispensaryName",
|
||||
dispensary_slug AS "dispensarySlug",
|
||||
state,
|
||||
city,
|
||||
menu_type AS "menuType",
|
||||
crawl_status AS "crawlStatus",
|
||||
last_crawl_at AS "lastCrawlAt",
|
||||
product_count AS "productCount",
|
||||
in_stock_count AS "inStockCount",
|
||||
brand_count AS "brandCount",
|
||||
avg_price AS "avgPrice",
|
||||
special_count AS "specialCount"
|
||||
FROM v_store_state_summary
|
||||
${whereClause}
|
||||
ORDER BY ${sortColumn} ${sortDir === 'asc' ? 'ASC' : 'DESC'} NULLS LAST
|
||||
LIMIT $2 OFFSET $3
|
||||
`, [state, limit, offset]);
|
||||
|
||||
return result.rows;
|
||||
}
|
||||
|
||||
// =========================================================================
|
||||
// Price Analytics
|
||||
// =========================================================================
|
||||
|
||||
/**
|
||||
* Get price distribution by state
|
||||
*/
|
||||
async getStorePriceDistribution(
|
||||
state: string,
|
||||
options: { category?: string; brandId?: number } = {}
|
||||
): Promise<StatePriceDistribution[]> {
|
||||
const { category, brandId } = options;
|
||||
|
||||
const result = await this.pool.query(`
|
||||
SELECT * FROM fn_national_price_comparison($1, $2)
|
||||
WHERE state = $3
|
||||
`, [category || null, brandId || null, state]);
|
||||
|
||||
return result.rows.map(row => ({
|
||||
state: row.state,
|
||||
stateName: row.state_name,
|
||||
productCount: parseInt(row.product_count),
|
||||
avgPrice: parseFloat(row.avg_price),
|
||||
minPrice: parseFloat(row.min_price),
|
||||
maxPrice: parseFloat(row.max_price),
|
||||
medianPrice: parseFloat(row.median_price),
|
||||
priceStddev: parseFloat(row.price_stddev),
|
||||
}));
|
||||
}
|
||||
|
||||
/**
|
||||
* Get national price comparison across all states
|
||||
*/
|
||||
async getNationalPriceComparison(
|
||||
options: { category?: string; brandId?: number } = {}
|
||||
): Promise<StatePriceDistribution[]> {
|
||||
const { category, brandId } = options;
|
||||
|
||||
const result = await this.pool.query(`
|
||||
SELECT * FROM fn_national_price_comparison($1, $2)
|
||||
`, [category || null, brandId || null]);
|
||||
|
||||
return result.rows.map(row => ({
|
||||
state: row.state,
|
||||
stateName: row.state_name,
|
||||
productCount: parseInt(row.product_count),
|
||||
avgPrice: parseFloat(row.avg_price),
|
||||
minPrice: parseFloat(row.min_price),
|
||||
maxPrice: parseFloat(row.max_price),
|
||||
medianPrice: parseFloat(row.median_price),
|
||||
priceStddev: parseFloat(row.price_stddev),
|
||||
}));
|
||||
}
|
||||
|
||||
// =========================================================================
|
||||
// National Analytics
|
||||
// =========================================================================
|
||||
|
||||
/**
|
||||
* Get national summary across all states
|
||||
*/
|
||||
async getNationalSummary(): Promise<NationalSummary> {
|
||||
const stateMetrics = await this.getAllStateMetrics();
|
||||
|
||||
const result = await this.pool.query(`
|
||||
SELECT
|
||||
COUNT(DISTINCT s.code) AS total_states,
|
||||
COUNT(DISTINCT CASE WHEN EXISTS (
|
||||
SELECT 1 FROM dispensaries d WHERE d.state = s.code AND d.menu_type IS NOT NULL
|
||||
) THEN s.code END) AS active_states,
|
||||
(SELECT COUNT(*) FROM dispensaries WHERE state IS NOT NULL) AS total_stores,
|
||||
(SELECT COUNT(*) FROM store_products sp
|
||||
JOIN dispensaries d ON sp.dispensary_id = d.id
|
||||
WHERE d.state IS NOT NULL) AS total_products,
|
||||
(SELECT COUNT(DISTINCT brand_id) FROM store_products sp
|
||||
JOIN dispensaries d ON sp.dispensary_id = d.id
|
||||
WHERE d.state IS NOT NULL AND sp.brand_id IS NOT NULL) AS total_brands,
|
||||
(SELECT AVG(price_rec) FROM store_products sp
|
||||
JOIN dispensaries d ON sp.dispensary_id = d.id
|
||||
WHERE d.state IS NOT NULL AND sp.price_rec > 0) AS avg_price_national
|
||||
FROM states s
|
||||
`);
|
||||
|
||||
const data = result.rows[0];
|
||||
|
||||
return {
|
||||
totalStates: parseInt(data.total_states),
|
||||
activeStates: parseInt(data.active_states),
|
||||
totalStores: parseInt(data.total_stores),
|
||||
totalProducts: parseInt(data.total_products),
|
||||
totalBrands: parseInt(data.total_brands),
|
||||
avgPriceNational: data.avg_price_national
|
||||
? Math.round(parseFloat(data.avg_price_national) * 100) / 100
|
||||
: null,
|
||||
stateMetrics,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Get heatmap data for a specific metric
|
||||
*/
|
||||
async getStateHeatmapData(
|
||||
metric: 'stores' | 'products' | 'brands' | 'avgPrice' | 'penetration',
|
||||
options: { brandId?: number; category?: string } = {}
|
||||
): Promise<StateHeatmapData[]> {
|
||||
let query: string;
|
||||
let params: any[] = [];
|
||||
|
||||
switch (metric) {
|
||||
case 'stores':
|
||||
query = `
|
||||
SELECT state, state_name AS "stateName", dispensary_count AS value, 'stores' AS label
|
||||
FROM mv_state_metrics
|
||||
WHERE state IS NOT NULL
|
||||
ORDER BY state
|
||||
`;
|
||||
break;
|
||||
|
||||
case 'products':
|
||||
query = `
|
||||
SELECT state, state_name AS "stateName", total_products AS value, 'products' AS label
|
||||
FROM mv_state_metrics
|
||||
WHERE state IS NOT NULL
|
||||
ORDER BY state
|
||||
`;
|
||||
break;
|
||||
|
||||
case 'brands':
|
||||
query = `
|
||||
SELECT state, state_name AS "stateName", unique_brands AS value, 'brands' AS label
|
||||
FROM mv_state_metrics
|
||||
WHERE state IS NOT NULL
|
||||
ORDER BY state
|
||||
`;
|
||||
break;
|
||||
|
||||
case 'avgPrice':
|
||||
query = `
|
||||
SELECT state, state_name AS "stateName", avg_price_rec AS value, 'avg price' AS label
|
||||
FROM mv_state_metrics
|
||||
WHERE state IS NOT NULL AND avg_price_rec IS NOT NULL
|
||||
ORDER BY state
|
||||
`;
|
||||
break;
|
||||
|
||||
case 'penetration':
|
||||
if (!options.brandId) {
|
||||
throw new Error('brandId required for penetration heatmap');
|
||||
}
|
||||
query = `
|
||||
SELECT state, state_name AS "stateName", penetration_pct AS value, 'penetration %' AS label
|
||||
FROM fn_brand_state_penetration($1)
|
||||
ORDER BY state
|
||||
`;
|
||||
params = [options.brandId];
|
||||
break;
|
||||
|
||||
default:
|
||||
throw new Error(`Unknown metric: ${metric}`);
|
||||
}
|
||||
|
||||
const result = await this.pool.query(query, params);
|
||||
return result.rows;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get national penetration trend for a brand
|
||||
*/
|
||||
async getNationalPenetrationTrend(
|
||||
brandId: number,
|
||||
options: { days?: number } = {}
|
||||
): Promise<NationalPenetrationTrend> {
|
||||
const { days = 30 } = options;
|
||||
|
||||
// Get brand info
|
||||
const brandResult = await this.pool.query(`
|
||||
SELECT id, name FROM brands WHERE id = $1
|
||||
`, [brandId]);
|
||||
|
||||
if (brandResult.rows.length === 0) {
|
||||
throw new Error(`Brand ${brandId} not found`);
|
||||
}
|
||||
|
||||
// Get historical data from snapshots
|
||||
const result = await this.pool.query(`
|
||||
WITH daily_presence AS (
|
||||
SELECT
|
||||
DATE(sps.captured_at) AS date,
|
||||
COUNT(DISTINCT d.state) AS states_present,
|
||||
COUNT(DISTINCT d.id) AS stores_with_brand
|
||||
FROM store_product_snapshots sps
|
||||
JOIN dispensaries d ON sps.dispensary_id = d.id
|
||||
JOIN store_products sp ON sps.store_product_id = sp.id
|
||||
WHERE sp.brand_id = $1
|
||||
AND sps.captured_at > NOW() - INTERVAL '1 day' * $2
|
||||
AND d.state IS NOT NULL
|
||||
GROUP BY DATE(sps.captured_at)
|
||||
),
|
||||
daily_totals AS (
|
||||
SELECT
|
||||
DATE(sps.captured_at) AS date,
|
||||
COUNT(DISTINCT d.id) AS total_stores
|
||||
FROM store_product_snapshots sps
|
||||
JOIN dispensaries d ON sps.dispensary_id = d.id
|
||||
WHERE sps.captured_at > NOW() - INTERVAL '1 day' * $2
|
||||
AND d.state IS NOT NULL
|
||||
GROUP BY DATE(sps.captured_at)
|
||||
)
|
||||
SELECT
|
||||
dp.date,
|
||||
dp.states_present,
|
||||
dt.total_stores,
|
||||
ROUND(dp.stores_with_brand::NUMERIC / NULLIF(dt.total_stores, 0) * 100, 2) AS penetration_pct
|
||||
FROM daily_presence dp
|
||||
JOIN daily_totals dt ON dp.date = dt.date
|
||||
ORDER BY dp.date
|
||||
`, [brandId, days]);
|
||||
|
||||
return {
|
||||
brandId,
|
||||
brandName: brandResult.rows[0].name,
|
||||
dataPoints: result.rows.map(row => ({
|
||||
date: row.date.toISOString().split('T')[0],
|
||||
statesPresent: parseInt(row.states_present),
|
||||
totalStores: parseInt(row.total_stores),
|
||||
penetrationPct: parseFloat(row.penetration_pct || '0'),
|
||||
})),
|
||||
};
|
||||
}
|
||||
|
||||
// =========================================================================
|
||||
// Utility Methods
|
||||
// =========================================================================
|
||||
|
||||
/**
|
||||
* Refresh materialized views
|
||||
* Uses direct REFRESH MATERIALIZED VIEW for compatibility
|
||||
*/
|
||||
async refreshMetrics(): Promise<void> {
|
||||
// Use direct refresh command instead of function call for better compatibility
|
||||
// CONCURRENTLY requires a unique index (idx_mv_state_metrics_state exists)
|
||||
await this.pool.query('REFRESH MATERIALIZED VIEW CONCURRENTLY mv_state_metrics');
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate state code
|
||||
*/
|
||||
async isValidState(state: string): Promise<boolean> {
|
||||
const result = await this.pool.query(`
|
||||
SELECT 1 FROM states WHERE code = $1
|
||||
`, [state]);
|
||||
return result.rows.length > 0;
|
||||
}
|
||||
}
|
||||
199
backend/src/multi-state/types.ts
Normal file
199
backend/src/multi-state/types.ts
Normal file
@@ -0,0 +1,199 @@
|
||||
/**
|
||||
* Multi-State Module Types
|
||||
* Phase 4: Multi-State Expansion
|
||||
*/
|
||||
|
||||
// Core state types
|
||||
export interface State {
|
||||
code: string;
|
||||
name: string;
|
||||
}
|
||||
|
||||
export interface StateMetrics {
|
||||
state: string;
|
||||
stateName: string;
|
||||
storeCount: number;
|
||||
dutchieStores: number;
|
||||
activeStores: number;
|
||||
totalProducts: number;
|
||||
inStockProducts: number;
|
||||
onSpecialProducts: number;
|
||||
uniqueBrands: number;
|
||||
uniqueCategories: number;
|
||||
avgPriceRec: number | null;
|
||||
minPriceRec: number | null;
|
||||
maxPriceRec: number | null;
|
||||
refreshedAt: Date;
|
||||
}
|
||||
|
||||
export interface StateSummary extends StateMetrics {
|
||||
recentCrawls: number;
|
||||
failedCrawls: number;
|
||||
lastCrawlAt: Date | null;
|
||||
topBrands: BrandInState[];
|
||||
topCategories: CategoryInState[];
|
||||
}
|
||||
|
||||
// Brand analytics
|
||||
export interface BrandInState {
|
||||
brandId: number;
|
||||
brandName: string;
|
||||
brandSlug: string;
|
||||
storeCount: number;
|
||||
productCount: number;
|
||||
avgPrice: number | null;
|
||||
firstSeenInState: Date | null;
|
||||
lastSeenInState: Date | null;
|
||||
}
|
||||
|
||||
export interface BrandStatePenetration {
|
||||
state: string;
|
||||
stateName: string;
|
||||
totalStores: number;
|
||||
storesWithBrand: number;
|
||||
penetrationPct: number;
|
||||
productCount: number;
|
||||
avgPrice: number | null;
|
||||
}
|
||||
|
||||
export interface BrandCrossStateComparison {
|
||||
brandId: number;
|
||||
brandName: string;
|
||||
states: BrandStatePenetration[];
|
||||
nationalPenetration: number;
|
||||
nationalAvgPrice: number | null;
|
||||
bestPerformingState: string | null;
|
||||
worstPerformingState: string | null;
|
||||
}
|
||||
|
||||
// Category analytics
|
||||
export interface CategoryInState {
|
||||
category: string;
|
||||
productCount: number;
|
||||
storeCount: number;
|
||||
avgPrice: number | null;
|
||||
inStockCount: number;
|
||||
onSpecialCount: number;
|
||||
}
|
||||
|
||||
export interface CategoryStateDist {
|
||||
state: string;
|
||||
stateName: string;
|
||||
category: string;
|
||||
productCount: number;
|
||||
storeCount: number;
|
||||
avgPrice: number | null;
|
||||
marketShare: number;
|
||||
}
|
||||
|
||||
export interface CategoryCrossStateComparison {
|
||||
category: string;
|
||||
states: CategoryStateDist[];
|
||||
nationalProductCount: number;
|
||||
nationalAvgPrice: number | null;
|
||||
dominantState: string | null;
|
||||
}
|
||||
|
||||
// Store analytics
|
||||
export interface StoreInState {
|
||||
dispensaryId: number;
|
||||
dispensaryName: string;
|
||||
dispensarySlug: string;
|
||||
state: string;
|
||||
city: string;
|
||||
menuType: string | null;
|
||||
crawlStatus: string;
|
||||
lastCrawlAt: Date | null;
|
||||
productCount: number;
|
||||
inStockCount: number;
|
||||
brandCount: number;
|
||||
avgPrice: number | null;
|
||||
specialCount: number;
|
||||
}
|
||||
|
||||
// Price analytics
|
||||
export interface StatePriceDistribution {
|
||||
state: string;
|
||||
stateName: string;
|
||||
productCount: number;
|
||||
avgPrice: number;
|
||||
minPrice: number;
|
||||
maxPrice: number;
|
||||
medianPrice: number;
|
||||
priceStddev: number;
|
||||
}
|
||||
|
||||
export interface NationalPriceTrend {
|
||||
date: string;
|
||||
states: {
|
||||
[stateCode: string]: {
|
||||
avgPrice: number;
|
||||
productCount: number;
|
||||
};
|
||||
};
|
||||
}
|
||||
|
||||
export interface ProductPriceByState {
|
||||
productId: string;
|
||||
productName: string;
|
||||
states: {
|
||||
state: string;
|
||||
stateName: string;
|
||||
avgPrice: number;
|
||||
minPrice: number;
|
||||
maxPrice: number;
|
||||
storeCount: number;
|
||||
}[];
|
||||
}
|
||||
|
||||
// National metrics
|
||||
export interface NationalSummary {
|
||||
totalStates: number;
|
||||
activeStates: number;
|
||||
totalStores: number;
|
||||
totalProducts: number;
|
||||
totalBrands: number;
|
||||
avgPriceNational: number | null;
|
||||
stateMetrics: StateMetrics[];
|
||||
}
|
||||
|
||||
export interface NationalPenetrationTrend {
|
||||
brandId: number;
|
||||
brandName: string;
|
||||
dataPoints: {
|
||||
date: string;
|
||||
statesPresent: number;
|
||||
totalStores: number;
|
||||
penetrationPct: number;
|
||||
}[];
|
||||
}
|
||||
|
||||
// Heatmap data
|
||||
export interface StateHeatmapData {
|
||||
state: string;
|
||||
stateName: string;
|
||||
value: number;
|
||||
label: string;
|
||||
}
|
||||
|
||||
// Query options
|
||||
export interface StateQueryOptions {
|
||||
state?: string;
|
||||
states?: string[];
|
||||
includeInactive?: boolean;
|
||||
limit?: number;
|
||||
offset?: number;
|
||||
sortBy?: string;
|
||||
sortDir?: 'asc' | 'desc';
|
||||
dateFrom?: Date;
|
||||
dateTo?: Date;
|
||||
}
|
||||
|
||||
export interface CrossStateQueryOptions {
|
||||
states: string[];
|
||||
metric: 'price' | 'penetration' | 'products' | 'stores';
|
||||
category?: string;
|
||||
brandId?: number;
|
||||
dateFrom?: Date;
|
||||
dateTo?: Date;
|
||||
}
|
||||
@@ -26,6 +26,21 @@ import { DutchieAZStores } from './pages/DutchieAZStores';
|
||||
import { DutchieAZStoreDetail } from './pages/DutchieAZStoreDetail';
|
||||
import { WholesaleAnalytics } from './pages/WholesaleAnalytics';
|
||||
import { Users } from './pages/Users';
|
||||
import { OrchestratorDashboard } from './pages/OrchestratorDashboard';
|
||||
import { OrchestratorProducts } from './pages/OrchestratorProducts';
|
||||
import { OrchestratorBrands } from './pages/OrchestratorBrands';
|
||||
import { OrchestratorStores } from './pages/OrchestratorStores';
|
||||
import { ChainsDashboard } from './pages/ChainsDashboard';
|
||||
import { IntelligenceBrands } from './pages/IntelligenceBrands';
|
||||
import { IntelligencePricing } from './pages/IntelligencePricing';
|
||||
import { IntelligenceStores } from './pages/IntelligenceStores';
|
||||
import { SyncInfoPanel } from './pages/SyncInfoPanel';
|
||||
import NationalDashboard from './pages/NationalDashboard';
|
||||
import StateHeatmap from './pages/StateHeatmap';
|
||||
import CrossStateCompare from './pages/CrossStateCompare';
|
||||
import { Discovery } from './pages/Discovery';
|
||||
import { WorkersDashboard } from './pages/WorkersDashboard';
|
||||
import { ScraperOverviewDashboard } from './pages/ScraperOverviewDashboard';
|
||||
import { PrivateRoute } from './components/PrivateRoute';
|
||||
|
||||
export default function App() {
|
||||
@@ -59,6 +74,29 @@ export default function App() {
|
||||
<Route path="/api-permissions" element={<PrivateRoute><ApiPermissions /></PrivateRoute>} />
|
||||
<Route path="/wholesale-analytics" element={<PrivateRoute><WholesaleAnalytics /></PrivateRoute>} />
|
||||
<Route path="/users" element={<PrivateRoute><Users /></PrivateRoute>} />
|
||||
{/* National / Multi-State routes */}
|
||||
<Route path="/national" element={<PrivateRoute><NationalDashboard /></PrivateRoute>} />
|
||||
<Route path="/national/heatmap" element={<PrivateRoute><StateHeatmap /></PrivateRoute>} />
|
||||
<Route path="/national/compare" element={<PrivateRoute><CrossStateCompare /></PrivateRoute>} />
|
||||
{/* Admin routes */}
|
||||
<Route path="/admin" element={<Navigate to="/admin/orchestrator" replace />} />
|
||||
<Route path="/admin/orchestrator" element={<PrivateRoute><OrchestratorDashboard /></PrivateRoute>} />
|
||||
<Route path="/admin/orchestrator/products" element={<PrivateRoute><OrchestratorProducts /></PrivateRoute>} />
|
||||
<Route path="/admin/orchestrator/brands" element={<PrivateRoute><OrchestratorBrands /></PrivateRoute>} />
|
||||
<Route path="/admin/orchestrator/stores" element={<PrivateRoute><OrchestratorStores /></PrivateRoute>} />
|
||||
<Route path="/admin/orchestrator/chains" element={<PrivateRoute><ChainsDashboard /></PrivateRoute>} />
|
||||
{/* Intelligence routes */}
|
||||
<Route path="/admin/intelligence" element={<Navigate to="/admin/intelligence/brands" replace />} />
|
||||
<Route path="/admin/intelligence/brands" element={<PrivateRoute><IntelligenceBrands /></PrivateRoute>} />
|
||||
<Route path="/admin/intelligence/pricing" element={<PrivateRoute><IntelligencePricing /></PrivateRoute>} />
|
||||
<Route path="/admin/intelligence/stores" element={<PrivateRoute><IntelligenceStores /></PrivateRoute>} />
|
||||
<Route path="/admin/intelligence/sync" element={<PrivateRoute><SyncInfoPanel /></PrivateRoute>} />
|
||||
{/* Discovery routes */}
|
||||
<Route path="/discovery" element={<PrivateRoute><Discovery /></PrivateRoute>} />
|
||||
{/* Workers Dashboard */}
|
||||
<Route path="/workers" element={<PrivateRoute><WorkersDashboard /></PrivateRoute>} />
|
||||
{/* Scraper Overview Dashboard (new primary) */}
|
||||
<Route path="/scraper/overview" element={<PrivateRoute><ScraperOverviewDashboard /></PrivateRoute>} />
|
||||
<Route path="*" element={<Navigate to="/dashboard" replace />} />
|
||||
</Routes>
|
||||
</BrowserRouter>
|
||||
|
||||
@@ -2,6 +2,7 @@ import { ReactNode, useEffect, useState } from 'react';
|
||||
import { useNavigate, useLocation } from 'react-router-dom';
|
||||
import { useAuthStore } from '../store/authStore';
|
||||
import { api } from '../lib/api';
|
||||
import { StateSelector } from './StateSelector';
|
||||
import {
|
||||
LayoutDashboard,
|
||||
Store,
|
||||
@@ -20,7 +21,13 @@ import {
|
||||
LogOut,
|
||||
CheckCircle,
|
||||
Key,
|
||||
Users
|
||||
Users,
|
||||
Globe,
|
||||
Map,
|
||||
Search,
|
||||
HardHat,
|
||||
Gauge,
|
||||
Archive
|
||||
} from 'lucide-react';
|
||||
|
||||
interface LayoutProps {
|
||||
@@ -132,6 +139,11 @@ export function Layout({ children }: LayoutProps) {
|
||||
<p className="text-xs text-gray-500 mt-2">{user?.email}</p>
|
||||
</div>
|
||||
|
||||
{/* State Selector */}
|
||||
<div className="px-4 py-3 border-b border-gray-200 bg-gray-50">
|
||||
<StateSelector showLabel={false} />
|
||||
</div>
|
||||
|
||||
{/* Navigation */}
|
||||
<nav className="flex-1 px-3 py-4 space-y-6">
|
||||
<NavSection title="Main">
|
||||
@@ -173,7 +185,28 @@ export function Layout({ children }: LayoutProps) {
|
||||
/>
|
||||
</NavSection>
|
||||
|
||||
<NavSection title="AZ Data">
|
||||
<NavSection title="National">
|
||||
<NavLink
|
||||
to="/national"
|
||||
icon={<Globe className="w-4 h-4" />}
|
||||
label="National Dashboard"
|
||||
isActive={isActive('/national', true)}
|
||||
/>
|
||||
<NavLink
|
||||
to="/national/heatmap"
|
||||
icon={<Map className="w-4 h-4" />}
|
||||
label="State Heatmap"
|
||||
isActive={isActive('/national/heatmap')}
|
||||
/>
|
||||
<NavLink
|
||||
to="/national/compare"
|
||||
icon={<TrendingUp className="w-4 h-4" />}
|
||||
label="Cross-State Compare"
|
||||
isActive={isActive('/national/compare')}
|
||||
/>
|
||||
</NavSection>
|
||||
|
||||
<NavSection title="State Data">
|
||||
<NavLink
|
||||
to="/wholesale-analytics"
|
||||
icon={<TrendingUp className="w-4 h-4" />}
|
||||
@@ -183,35 +216,44 @@ export function Layout({ children }: LayoutProps) {
|
||||
<NavLink
|
||||
to="/az"
|
||||
icon={<Store className="w-4 h-4" />}
|
||||
label="AZ Stores"
|
||||
label="Stores"
|
||||
isActive={isActive('/az', false)}
|
||||
/>
|
||||
<NavLink
|
||||
to="/az-schedule"
|
||||
icon={<Calendar className="w-4 h-4" />}
|
||||
label="AZ Schedule"
|
||||
label="Crawl Schedule"
|
||||
isActive={isActive('/az-schedule')}
|
||||
/>
|
||||
</NavSection>
|
||||
|
||||
<NavSection title="Scraper">
|
||||
<NavLink
|
||||
to="/scraper-tools"
|
||||
icon={<Wrench className="w-4 h-4" />}
|
||||
label="Tools"
|
||||
isActive={isActive('/scraper-tools')}
|
||||
to="/scraper/overview"
|
||||
icon={<Gauge className="w-4 h-4" />}
|
||||
label="Dashboard"
|
||||
isActive={isActive('/scraper/overview')}
|
||||
/>
|
||||
<NavLink
|
||||
to="/scraper-schedule"
|
||||
icon={<Clock className="w-4 h-4" />}
|
||||
label="Schedule"
|
||||
isActive={isActive('/scraper-schedule')}
|
||||
to="/workers"
|
||||
icon={<HardHat className="w-4 h-4" />}
|
||||
label="Workers"
|
||||
isActive={isActive('/workers')}
|
||||
/>
|
||||
<NavLink
|
||||
to="/scraper-monitor"
|
||||
to="/discovery"
|
||||
icon={<Search className="w-4 h-4" />}
|
||||
label="Store Discovery"
|
||||
isActive={isActive('/discovery')}
|
||||
/>
|
||||
</NavSection>
|
||||
|
||||
<NavSection title="Orchestrator">
|
||||
<NavLink
|
||||
to="/admin/orchestrator"
|
||||
icon={<Activity className="w-4 h-4" />}
|
||||
label="Monitor"
|
||||
isActive={isActive('/scraper-monitor')}
|
||||
label="Dashboard"
|
||||
isActive={isActive('/admin/orchestrator')}
|
||||
/>
|
||||
</NavSection>
|
||||
|
||||
@@ -253,6 +295,27 @@ export function Layout({ children }: LayoutProps) {
|
||||
isActive={isActive('/users')}
|
||||
/>
|
||||
</NavSection>
|
||||
|
||||
<NavSection title="Legacy">
|
||||
<NavLink
|
||||
to="/scraper-tools"
|
||||
icon={<Archive className="w-4 h-4" />}
|
||||
label="Tools"
|
||||
isActive={isActive('/scraper-tools')}
|
||||
/>
|
||||
<NavLink
|
||||
to="/scraper-schedule"
|
||||
icon={<Archive className="w-4 h-4" />}
|
||||
label="Schedule"
|
||||
isActive={isActive('/scraper-schedule')}
|
||||
/>
|
||||
<NavLink
|
||||
to="/scraper-monitor"
|
||||
icon={<Archive className="w-4 h-4" />}
|
||||
label="Monitor"
|
||||
isActive={isActive('/scraper-monitor')}
|
||||
/>
|
||||
</NavSection>
|
||||
</nav>
|
||||
|
||||
{/* Logout */}
|
||||
|
||||
1115
cannaiq/src/components/StoreOrchestratorPanel.tsx
Normal file
1115
cannaiq/src/components/StoreOrchestratorPanel.tsx
Normal file
File diff suppressed because it is too large
Load Diff
138
cannaiq/src/components/WorkerRoleBadge.tsx
Normal file
138
cannaiq/src/components/WorkerRoleBadge.tsx
Normal file
@@ -0,0 +1,138 @@
|
||||
/**
|
||||
* WorkerRoleBadge Component
|
||||
*
|
||||
* Displays a badge for worker roles with consistent styling.
|
||||
*
|
||||
* Role mapping:
|
||||
* product_sync / visibility_audit → [Products]
|
||||
* store_discovery → [Discovery]
|
||||
* entry_point_finder → [End Points]
|
||||
* analytics_refresh → [Analytics]
|
||||
* Unknown → [Other]
|
||||
*/
|
||||
|
||||
import React from 'react';
|
||||
|
||||
interface WorkerRoleBadgeProps {
|
||||
role: string | null | undefined;
|
||||
size?: 'sm' | 'md';
|
||||
}
|
||||
|
||||
interface RoleConfig {
|
||||
label: string;
|
||||
bg: string;
|
||||
color: string;
|
||||
}
|
||||
|
||||
const roleMapping: Record<string, RoleConfig> = {
|
||||
product_sync: {
|
||||
label: 'Products',
|
||||
bg: '#dbeafe',
|
||||
color: '#1e40af',
|
||||
},
|
||||
visibility_audit: {
|
||||
label: 'Products',
|
||||
bg: '#dbeafe',
|
||||
color: '#1e40af',
|
||||
},
|
||||
store_discovery: {
|
||||
label: 'Discovery',
|
||||
bg: '#d1fae5',
|
||||
color: '#065f46',
|
||||
},
|
||||
entry_point_finder: {
|
||||
label: 'End Points',
|
||||
bg: '#fef3c7',
|
||||
color: '#92400e',
|
||||
},
|
||||
analytics_refresh: {
|
||||
label: 'Analytics',
|
||||
bg: '#ede9fe',
|
||||
color: '#5b21b6',
|
||||
},
|
||||
};
|
||||
|
||||
const defaultConfig: RoleConfig = {
|
||||
label: 'Other',
|
||||
bg: '#f3f4f6',
|
||||
color: '#374151',
|
||||
};
|
||||
|
||||
export function getRoleConfig(role: string | null | undefined): RoleConfig {
|
||||
if (!role) return defaultConfig;
|
||||
return roleMapping[role] || defaultConfig;
|
||||
}
|
||||
|
||||
export function WorkerRoleBadge({ role, size = 'sm' }: WorkerRoleBadgeProps) {
|
||||
const config = getRoleConfig(role);
|
||||
|
||||
const fontSize = size === 'sm' ? '11px' : '12px';
|
||||
const padding = size === 'sm' ? '2px 8px' : '4px 10px';
|
||||
|
||||
return (
|
||||
<span
|
||||
style={{
|
||||
display: 'inline-block',
|
||||
padding,
|
||||
borderRadius: '12px',
|
||||
fontSize,
|
||||
fontWeight: 600,
|
||||
background: config.bg,
|
||||
color: config.color,
|
||||
whiteSpace: 'nowrap',
|
||||
}}
|
||||
>
|
||||
{config.label}
|
||||
</span>
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Format scope metadata for display
|
||||
*
|
||||
* @param metadata - Job metadata containing scope info
|
||||
* @returns Human-readable scope string
|
||||
*/
|
||||
export function formatScope(metadata: any): string {
|
||||
if (!metadata) return '-';
|
||||
|
||||
// Check for states scope
|
||||
if (metadata.states && Array.isArray(metadata.states)) {
|
||||
if (metadata.states.length <= 5) {
|
||||
return metadata.states.join(', ');
|
||||
}
|
||||
return `${metadata.states.slice(0, 3).join(', ')} +${metadata.states.length - 3} more`;
|
||||
}
|
||||
|
||||
// Check for storeIds scope
|
||||
if (metadata.storeIds && Array.isArray(metadata.storeIds)) {
|
||||
return `${metadata.storeIds.length} stores (custom scope)`;
|
||||
}
|
||||
|
||||
// Check for dispensaryIds scope
|
||||
if (metadata.dispensaryIds && Array.isArray(metadata.dispensaryIds)) {
|
||||
return `${metadata.dispensaryIds.length} stores (custom scope)`;
|
||||
}
|
||||
|
||||
// Check for state field
|
||||
if (metadata.state) {
|
||||
return metadata.state;
|
||||
}
|
||||
|
||||
// Check for description
|
||||
if (metadata.description) {
|
||||
return metadata.description;
|
||||
}
|
||||
|
||||
// Check for scope.states
|
||||
if (metadata.scope?.states && Array.isArray(metadata.scope.states)) {
|
||||
if (metadata.scope.states.length <= 5) {
|
||||
return metadata.scope.states.join(', ');
|
||||
}
|
||||
return `${metadata.scope.states.slice(0, 3).join(', ')} +${metadata.scope.states.length - 3} more`;
|
||||
}
|
||||
|
||||
return '-';
|
||||
}
|
||||
|
||||
export default WorkerRoleBadge;
|
||||
File diff suppressed because it is too large
Load Diff
378
cannaiq/src/pages/NationalDashboard.tsx
Normal file
378
cannaiq/src/pages/NationalDashboard.tsx
Normal file
@@ -0,0 +1,378 @@
|
||||
/**
|
||||
* National Dashboard
|
||||
*
|
||||
* Multi-state overview with key metrics and state comparison.
|
||||
* Phase 4: Multi-State Expansion
|
||||
*/
|
||||
|
||||
import { useState, useEffect } from 'react';
|
||||
import { useNavigate } from 'react-router-dom';
|
||||
import { Layout } from '../components/Layout';
|
||||
import { StateBadge } from '../components/StateSelector';
|
||||
import { useStateStore } from '../store/stateStore';
|
||||
import { api } from '../lib/api';
|
||||
import {
|
||||
Globe,
|
||||
Store,
|
||||
Package,
|
||||
Tag,
|
||||
TrendingUp,
|
||||
TrendingDown,
|
||||
DollarSign,
|
||||
MapPin,
|
||||
ArrowRight,
|
||||
RefreshCw,
|
||||
AlertCircle
|
||||
} from 'lucide-react';
|
||||
|
||||
interface StateMetric {
|
||||
state: string;
|
||||
stateName: string;
|
||||
storeCount: number;
|
||||
totalProducts: number;
|
||||
uniqueBrands: number;
|
||||
avgPriceRec: number | string | null;
|
||||
avgPriceMed?: number | string | null;
|
||||
inStockProducts: number;
|
||||
onSpecialProducts: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Safe money formatter that handles null, undefined, strings, and invalid numbers
|
||||
* @param value - Any value that might be a price
|
||||
* @param fallback - What to show when value is not usable (default: '—')
|
||||
* @returns Formatted price string like "$12.99" or the fallback
|
||||
*/
|
||||
function formatMoney(value: unknown, fallback = '—'): string {
|
||||
if (value === null || value === undefined) {
|
||||
return fallback;
|
||||
}
|
||||
|
||||
// Try to convert to number
|
||||
const num = typeof value === 'string' ? parseFloat(value) : Number(value);
|
||||
|
||||
// Check if it's a valid finite number
|
||||
if (!Number.isFinite(num)) {
|
||||
return fallback;
|
||||
}
|
||||
|
||||
return `$${num.toFixed(2)}`;
|
||||
}
|
||||
|
||||
interface NationalSummary {
|
||||
totalStates: number;
|
||||
activeStates: number;
|
||||
totalStores: number;
|
||||
totalProducts: number;
|
||||
totalBrands: number;
|
||||
avgPriceNational: number | null;
|
||||
stateMetrics: StateMetric[];
|
||||
}
|
||||
|
||||
function MetricCard({
|
||||
title,
|
||||
value,
|
||||
icon: Icon,
|
||||
trend,
|
||||
trendLabel,
|
||||
onClick,
|
||||
}: {
|
||||
title: string;
|
||||
value: string | number;
|
||||
icon: any;
|
||||
trend?: 'up' | 'down' | 'neutral';
|
||||
trendLabel?: string;
|
||||
onClick?: () => void;
|
||||
}) {
|
||||
return (
|
||||
<div
|
||||
onClick={onClick}
|
||||
className={`bg-white rounded-xl border border-gray-200 p-6 ${
|
||||
onClick ? 'cursor-pointer hover:border-emerald-300 hover:shadow-md transition-all' : ''
|
||||
}`}
|
||||
>
|
||||
<div className="flex items-center justify-between mb-4">
|
||||
<div className="w-10 h-10 bg-emerald-100 rounded-lg flex items-center justify-center">
|
||||
<Icon className="w-5 h-5 text-emerald-600" />
|
||||
</div>
|
||||
{trend && (
|
||||
<div className={`flex items-center gap-1 text-sm ${
|
||||
trend === 'up' ? 'text-green-600' : trend === 'down' ? 'text-red-600' : 'text-gray-500'
|
||||
}`}>
|
||||
{trend === 'up' ? <TrendingUp className="w-4 h-4" /> : trend === 'down' ? <TrendingDown className="w-4 h-4" /> : null}
|
||||
{trendLabel}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
<div className="text-2xl font-bold text-gray-900">{value}</div>
|
||||
<div className="text-sm text-gray-500 mt-1">{title}</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function StateRow({ metric, onClick }: { metric: StateMetric; onClick: () => void }) {
|
||||
return (
|
||||
<tr
|
||||
onClick={onClick}
|
||||
className="hover:bg-gray-50 cursor-pointer transition-colors"
|
||||
>
|
||||
<td className="px-4 py-3">
|
||||
<div className="flex items-center gap-2">
|
||||
<MapPin className="w-4 h-4 text-gray-400" />
|
||||
<span className="font-medium text-gray-900">{metric.stateName}</span>
|
||||
<span className="text-xs text-gray-500 bg-gray-100 px-1.5 py-0.5 rounded">
|
||||
{metric.state}
|
||||
</span>
|
||||
</div>
|
||||
</td>
|
||||
<td className="px-4 py-3 text-right">
|
||||
<span className="font-medium">{(metric.storeCount ?? 0).toLocaleString()}</span>
|
||||
</td>
|
||||
<td className="px-4 py-3 text-right">
|
||||
<span className="font-medium">{(metric.totalProducts ?? 0).toLocaleString()}</span>
|
||||
</td>
|
||||
<td className="px-4 py-3 text-right">
|
||||
<span className="font-medium">{(metric.uniqueBrands ?? 0).toLocaleString()}</span>
|
||||
</td>
|
||||
<td className="px-4 py-3 text-right">
|
||||
{formatMoney(metric.avgPriceRec, '—') !== '—' ? (
|
||||
<span className="font-medium text-emerald-600">
|
||||
{formatMoney(metric.avgPriceRec)}
|
||||
</span>
|
||||
) : (
|
||||
<span className="text-gray-400">—</span>
|
||||
)}
|
||||
</td>
|
||||
<td className="px-4 py-3 text-right">
|
||||
<span className={`px-2 py-1 rounded-full text-xs font-medium ${
|
||||
(metric.onSpecialProducts ?? 0) > 0 ? 'bg-orange-100 text-orange-700' : 'bg-gray-100 text-gray-600'
|
||||
}`}>
|
||||
{(metric.onSpecialProducts ?? 0).toLocaleString()} specials
|
||||
</span>
|
||||
</td>
|
||||
<td className="px-4 py-3 text-right">
|
||||
<ArrowRight className="w-4 h-4 text-gray-400" />
|
||||
</td>
|
||||
</tr>
|
||||
);
|
||||
}
|
||||
|
||||
export default function NationalDashboard() {
|
||||
const navigate = useNavigate();
|
||||
const { setSelectedState } = useStateStore();
|
||||
const [loading, setLoading] = useState(true);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
const [summary, setSummary] = useState<NationalSummary | null>(null);
|
||||
const [refreshing, setRefreshing] = useState(false);
|
||||
|
||||
const fetchData = async () => {
|
||||
setLoading(true);
|
||||
setError(null);
|
||||
try {
|
||||
const response = await api.get('/api/analytics/national/summary');
|
||||
if (response.data?.success && response.data.data) {
|
||||
setSummary(response.data.data);
|
||||
} else if (response.data?.totalStores !== undefined) {
|
||||
// Handle direct data format
|
||||
setSummary(response.data);
|
||||
}
|
||||
} catch (err: any) {
|
||||
setError(err.message || 'Failed to load national data');
|
||||
console.error('Failed to fetch national summary:', err);
|
||||
} finally {
|
||||
setLoading(false);
|
||||
}
|
||||
};
|
||||
|
||||
useEffect(() => {
|
||||
fetchData();
|
||||
}, []);
|
||||
|
||||
const handleRefreshMetrics = async () => {
|
||||
setRefreshing(true);
|
||||
try {
|
||||
await api.post('/api/admin/states/refresh-metrics');
|
||||
await fetchData();
|
||||
} catch (err) {
|
||||
console.error('Failed to refresh metrics:', err);
|
||||
} finally {
|
||||
setRefreshing(false);
|
||||
}
|
||||
};
|
||||
|
||||
const handleStateClick = (stateCode: string) => {
|
||||
setSelectedState(stateCode);
|
||||
navigate('/dashboard');
|
||||
};
|
||||
|
||||
if (loading) {
|
||||
return (
|
||||
<Layout>
|
||||
<div className="flex items-center justify-center h-64">
|
||||
<div className="text-gray-500">Loading national data...</div>
|
||||
</div>
|
||||
</Layout>
|
||||
);
|
||||
}
|
||||
|
||||
if (error) {
|
||||
return (
|
||||
<Layout>
|
||||
<div className="flex flex-col items-center justify-center h-64 gap-4">
|
||||
<AlertCircle className="w-12 h-12 text-red-500" />
|
||||
<div className="text-red-600">{error}</div>
|
||||
<button
|
||||
onClick={fetchData}
|
||||
className="px-4 py-2 bg-emerald-600 text-white rounded-lg hover:bg-emerald-700"
|
||||
>
|
||||
Retry
|
||||
</button>
|
||||
</div>
|
||||
</Layout>
|
||||
);
|
||||
}
|
||||
|
||||
return (
|
||||
<Layout>
|
||||
<div className="space-y-6">
|
||||
{/* Header */}
|
||||
<div className="flex items-center justify-between">
|
||||
<div>
|
||||
<h1 className="text-2xl font-bold text-gray-900">National Dashboard</h1>
|
||||
<p className="text-gray-500 mt-1">
|
||||
Multi-state cannabis market intelligence
|
||||
</p>
|
||||
</div>
|
||||
<div className="flex items-center gap-3">
|
||||
<StateBadge />
|
||||
<button
|
||||
onClick={handleRefreshMetrics}
|
||||
disabled={refreshing}
|
||||
className="flex items-center gap-2 px-3 py-2 text-sm text-gray-600 hover:text-gray-900 border border-gray-200 rounded-lg hover:bg-gray-50 disabled:opacity-50"
|
||||
>
|
||||
<RefreshCw className={`w-4 h-4 ${refreshing ? 'animate-spin' : ''}`} />
|
||||
Refresh Metrics
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Summary Cards */}
|
||||
{summary && (
|
||||
<>
|
||||
<div className="grid grid-cols-1 md:grid-cols-2 lg:grid-cols-4 gap-4">
|
||||
<MetricCard
|
||||
title="Active States"
|
||||
value={summary.activeStates}
|
||||
icon={Globe}
|
||||
/>
|
||||
<MetricCard
|
||||
title="Total Stores"
|
||||
value={(summary.totalStores ?? 0).toLocaleString()}
|
||||
icon={Store}
|
||||
/>
|
||||
<MetricCard
|
||||
title="Total Products"
|
||||
value={(summary.totalProducts ?? 0).toLocaleString()}
|
||||
icon={Package}
|
||||
/>
|
||||
<MetricCard
|
||||
title="National Avg Price"
|
||||
value={formatMoney(summary.avgPriceNational, '-')}
|
||||
icon={DollarSign}
|
||||
/>
|
||||
</div>
|
||||
|
||||
{/* States Table */}
|
||||
<div className="bg-white rounded-xl border border-gray-200 overflow-hidden">
|
||||
<div className="px-6 py-4 border-b border-gray-200">
|
||||
<h2 className="text-lg font-semibold text-gray-900">State Overview</h2>
|
||||
<p className="text-sm text-gray-500">Click a state to view detailed analytics</p>
|
||||
</div>
|
||||
<div className="overflow-x-auto">
|
||||
<table className="w-full">
|
||||
<thead className="bg-gray-50">
|
||||
<tr>
|
||||
<th className="px-4 py-3 text-left text-xs font-medium text-gray-500 uppercase tracking-wider">
|
||||
State
|
||||
</th>
|
||||
<th className="px-4 py-3 text-right text-xs font-medium text-gray-500 uppercase tracking-wider">
|
||||
Stores
|
||||
</th>
|
||||
<th className="px-4 py-3 text-right text-xs font-medium text-gray-500 uppercase tracking-wider">
|
||||
Products
|
||||
</th>
|
||||
<th className="px-4 py-3 text-right text-xs font-medium text-gray-500 uppercase tracking-wider">
|
||||
Brands
|
||||
</th>
|
||||
<th className="px-4 py-3 text-right text-xs font-medium text-gray-500 uppercase tracking-wider">
|
||||
Avg Price
|
||||
</th>
|
||||
<th className="px-4 py-3 text-right text-xs font-medium text-gray-500 uppercase tracking-wider">
|
||||
Specials
|
||||
</th>
|
||||
<th className="px-4 py-3 w-10"></th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody className="divide-y divide-gray-200">
|
||||
{summary.stateMetrics
|
||||
.filter(m => m.storeCount > 0)
|
||||
.sort((a, b) => b.totalProducts - a.totalProducts)
|
||||
.map((metric) => (
|
||||
<StateRow
|
||||
key={metric.state}
|
||||
metric={metric}
|
||||
onClick={() => handleStateClick(metric.state)}
|
||||
/>
|
||||
))}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Quick Links */}
|
||||
<div className="grid grid-cols-1 md:grid-cols-3 gap-4">
|
||||
<button
|
||||
onClick={() => navigate('/national/heatmap')}
|
||||
className="flex items-center gap-4 p-4 bg-white rounded-xl border border-gray-200 hover:border-emerald-300 hover:shadow-md transition-all text-left"
|
||||
>
|
||||
<div className="w-12 h-12 bg-blue-100 rounded-lg flex items-center justify-center">
|
||||
<Globe className="w-6 h-6 text-blue-600" />
|
||||
</div>
|
||||
<div>
|
||||
<div className="font-medium text-gray-900">State Heatmap</div>
|
||||
<div className="text-sm text-gray-500">Visualize metrics geographically</div>
|
||||
</div>
|
||||
</button>
|
||||
|
||||
<button
|
||||
onClick={() => navigate('/national/compare')}
|
||||
className="flex items-center gap-4 p-4 bg-white rounded-xl border border-gray-200 hover:border-emerald-300 hover:shadow-md transition-all text-left"
|
||||
>
|
||||
<div className="w-12 h-12 bg-purple-100 rounded-lg flex items-center justify-center">
|
||||
<TrendingUp className="w-6 h-6 text-purple-600" />
|
||||
</div>
|
||||
<div>
|
||||
<div className="font-medium text-gray-900">Cross-State Compare</div>
|
||||
<div className="text-sm text-gray-500">Compare brands & categories</div>
|
||||
</div>
|
||||
</button>
|
||||
|
||||
<button
|
||||
onClick={() => navigate('/analytics')}
|
||||
className="flex items-center gap-4 p-4 bg-white rounded-xl border border-gray-200 hover:border-emerald-300 hover:shadow-md transition-all text-left"
|
||||
>
|
||||
<div className="w-12 h-12 bg-emerald-100 rounded-lg flex items-center justify-center">
|
||||
<Tag className="w-6 h-6 text-emerald-600" />
|
||||
</div>
|
||||
<div>
|
||||
<div className="font-medium text-gray-900">Price Analytics</div>
|
||||
<div className="text-sm text-gray-500">National pricing trends</div>
|
||||
</div>
|
||||
</button>
|
||||
</div>
|
||||
</>
|
||||
)}
|
||||
</div>
|
||||
</Layout>
|
||||
);
|
||||
}
|
||||
472
cannaiq/src/pages/OrchestratorDashboard.tsx
Normal file
472
cannaiq/src/pages/OrchestratorDashboard.tsx
Normal file
@@ -0,0 +1,472 @@
|
||||
import { useEffect, useState } from 'react';
|
||||
import { useNavigate } from 'react-router-dom';
|
||||
import { Layout } from '../components/Layout';
|
||||
import { api } from '../lib/api';
|
||||
import {
|
||||
Package,
|
||||
Building2,
|
||||
CheckCircle,
|
||||
AlertTriangle,
|
||||
XCircle,
|
||||
RefreshCw,
|
||||
ChevronDown,
|
||||
Clock,
|
||||
FileText,
|
||||
Settings,
|
||||
Code,
|
||||
TrendingUp,
|
||||
TrendingDown,
|
||||
Minus,
|
||||
} from 'lucide-react';
|
||||
import { StoreOrchestratorPanel } from '../components/StoreOrchestratorPanel';
|
||||
|
||||
interface OrchestratorMetrics {
|
||||
total_products: number;
|
||||
total_brands: number;
|
||||
total_stores: number;
|
||||
market_sentiment: string;
|
||||
market_direction: string;
|
||||
healthy_count: number;
|
||||
sandbox_count: number;
|
||||
needs_manual_count: number;
|
||||
failing_count: number;
|
||||
}
|
||||
|
||||
interface StateInfo {
|
||||
state: string;
|
||||
storeCount: number;
|
||||
}
|
||||
|
||||
interface StoreInfo {
|
||||
id: number;
|
||||
name: string;
|
||||
city: string;
|
||||
state: string;
|
||||
provider: string;
|
||||
provider_raw?: string | null;
|
||||
provider_display?: string;
|
||||
platformDispensaryId: string | null;
|
||||
status: string;
|
||||
profileId: number | null;
|
||||
profileKey: string | null;
|
||||
sandboxAttempts: number;
|
||||
nextRetryAt: string | null;
|
||||
lastCrawlAt: string | null;
|
||||
lastSuccessAt: string | null;
|
||||
lastFailureAt: string | null;
|
||||
failedAt: string | null;
|
||||
consecutiveFailures: number;
|
||||
productCount: number;
|
||||
}
|
||||
|
||||
export function OrchestratorDashboard() {
|
||||
const navigate = useNavigate();
|
||||
const [metrics, setMetrics] = useState<OrchestratorMetrics | null>(null);
|
||||
const [states, setStates] = useState<StateInfo[]>([]);
|
||||
const [stores, setStores] = useState<StoreInfo[]>([]);
|
||||
const [totalStores, setTotalStores] = useState(0);
|
||||
const [loading, setLoading] = useState(true);
|
||||
const [selectedState, setSelectedState] = useState<string>('all');
|
||||
const [autoRefresh, setAutoRefresh] = useState(true);
|
||||
const [selectedStore, setSelectedStore] = useState<StoreInfo | null>(null);
|
||||
const [panelTab, setPanelTab] = useState<'control' | 'trace' | 'profile' | 'module' | 'debug'>('control');
|
||||
|
||||
useEffect(() => {
|
||||
loadData();
|
||||
|
||||
if (autoRefresh) {
|
||||
const interval = setInterval(loadData, 30000);
|
||||
return () => clearInterval(interval);
|
||||
}
|
||||
}, [autoRefresh, selectedState]);
|
||||
|
||||
const loadData = async () => {
|
||||
try {
|
||||
const [metricsData, statesData, storesData] = await Promise.all([
|
||||
api.getOrchestratorMetrics(),
|
||||
api.getOrchestratorStates(),
|
||||
api.getOrchestratorStores({ state: selectedState, limit: 200 }),
|
||||
]);
|
||||
|
||||
setMetrics(metricsData);
|
||||
setStates(statesData.states || []);
|
||||
setStores(storesData.stores || []);
|
||||
setTotalStores(storesData.total || 0);
|
||||
} catch (error) {
|
||||
console.error('Failed to load orchestrator data:', error);
|
||||
} finally {
|
||||
setLoading(false);
|
||||
}
|
||||
};
|
||||
|
||||
const getStatusPill = (status: string) => {
|
||||
switch (status) {
|
||||
case 'production':
|
||||
return (
|
||||
<span className="badge badge-success badge-sm gap-1">
|
||||
<CheckCircle className="w-3 h-3" />
|
||||
PRODUCTION
|
||||
</span>
|
||||
);
|
||||
case 'sandbox':
|
||||
return (
|
||||
<span className="badge badge-warning badge-sm gap-1">
|
||||
<Clock className="w-3 h-3" />
|
||||
SANDBOX
|
||||
</span>
|
||||
);
|
||||
case 'needs_manual':
|
||||
return (
|
||||
<span className="badge badge-error badge-sm gap-1">
|
||||
<AlertTriangle className="w-3 h-3" />
|
||||
NEEDS MANUAL
|
||||
</span>
|
||||
);
|
||||
case 'disabled':
|
||||
return (
|
||||
<span className="badge badge-ghost badge-sm gap-1">
|
||||
<XCircle className="w-3 h-3" />
|
||||
DISABLED
|
||||
</span>
|
||||
);
|
||||
case 'legacy':
|
||||
return (
|
||||
<span className="badge badge-outline badge-sm gap-1">
|
||||
LEGACY
|
||||
</span>
|
||||
);
|
||||
case 'pending':
|
||||
return (
|
||||
<span className="badge badge-info badge-sm gap-1">
|
||||
PENDING
|
||||
</span>
|
||||
);
|
||||
default:
|
||||
return (
|
||||
<span className="badge badge-ghost badge-sm">
|
||||
{status || 'UNKNOWN'}
|
||||
</span>
|
||||
);
|
||||
}
|
||||
};
|
||||
|
||||
const getMarketDirectionIcon = (direction: string) => {
|
||||
switch (direction) {
|
||||
case 'up':
|
||||
return <TrendingUp className="w-4 h-4 text-green-500" />;
|
||||
case 'down':
|
||||
return <TrendingDown className="w-4 h-4 text-red-500" />;
|
||||
default:
|
||||
return <Minus className="w-4 h-4 text-gray-400" />;
|
||||
}
|
||||
};
|
||||
|
||||
const formatTimeAgo = (dateStr: string | null) => {
|
||||
if (!dateStr) return '-';
|
||||
const date = new Date(dateStr);
|
||||
const now = new Date();
|
||||
const diff = now.getTime() - date.getTime();
|
||||
const hours = Math.floor(diff / (1000 * 60 * 60));
|
||||
const days = Math.floor(hours / 24);
|
||||
|
||||
if (days > 0) return `${days}d ago`;
|
||||
if (hours > 0) return `${hours}h ago`;
|
||||
const minutes = Math.floor(diff / (1000 * 60));
|
||||
if (minutes > 0) return `${minutes}m ago`;
|
||||
return 'just now';
|
||||
};
|
||||
|
||||
if (loading) {
|
||||
return (
|
||||
<Layout>
|
||||
<div className="text-center py-12">
|
||||
<div className="inline-block animate-spin rounded-full h-8 w-8 border-4 border-blue-500 border-t-transparent"></div>
|
||||
<p className="mt-2 text-sm text-gray-600">Loading orchestrator data...</p>
|
||||
</div>
|
||||
</Layout>
|
||||
);
|
||||
}
|
||||
|
||||
return (
|
||||
<Layout>
|
||||
<div className="space-y-6">
|
||||
{/* Header */}
|
||||
<div className="flex items-center justify-between">
|
||||
<div>
|
||||
<h1 className="text-2xl font-bold text-gray-900">Orchestrator Dashboard</h1>
|
||||
<p className="text-sm text-gray-600 mt-1">
|
||||
Crawler observability and per-store monitoring
|
||||
</p>
|
||||
</div>
|
||||
<div className="flex items-center gap-3">
|
||||
<label className="flex items-center gap-2 text-sm text-gray-600">
|
||||
<input
|
||||
type="checkbox"
|
||||
className="checkbox checkbox-sm"
|
||||
checked={autoRefresh}
|
||||
onChange={(e) => setAutoRefresh(e.target.checked)}
|
||||
/>
|
||||
Auto-refresh (30s)
|
||||
</label>
|
||||
<button
|
||||
onClick={loadData}
|
||||
className="btn btn-sm btn-outline gap-2"
|
||||
>
|
||||
<RefreshCw className="w-4 h-4" />
|
||||
Refresh
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Metrics Cards - Clickable */}
|
||||
{metrics && (
|
||||
<div className="grid grid-cols-2 md:grid-cols-4 lg:grid-cols-7 gap-4">
|
||||
<div
|
||||
onClick={() => navigate('/admin/orchestrator/products')}
|
||||
className="bg-white rounded-lg border border-gray-200 p-4 cursor-pointer hover:bg-blue-50 hover:border-blue-300 transition-colors"
|
||||
>
|
||||
<div className="flex items-center gap-2">
|
||||
<Package className="w-5 h-5 text-blue-500" />
|
||||
<div>
|
||||
<p className="text-xs text-gray-500">Products</p>
|
||||
<p className="text-lg font-bold">{metrics.total_products.toLocaleString()}</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div
|
||||
onClick={() => navigate('/admin/orchestrator/brands')}
|
||||
className="bg-white rounded-lg border border-gray-200 p-4 cursor-pointer hover:bg-purple-50 hover:border-purple-300 transition-colors"
|
||||
>
|
||||
<div className="flex items-center gap-2">
|
||||
<Building2 className="w-5 h-5 text-purple-500" />
|
||||
<div>
|
||||
<p className="text-xs text-gray-500">Brands</p>
|
||||
<p className="text-lg font-bold">{metrics.total_brands.toLocaleString()}</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div
|
||||
onClick={() => navigate('/admin/orchestrator/stores')}
|
||||
className="bg-white rounded-lg border border-gray-200 p-4 cursor-pointer hover:bg-gray-100 hover:border-gray-400 transition-colors"
|
||||
>
|
||||
<div className="flex items-center gap-2">
|
||||
<Building2 className="w-5 h-5 text-gray-500" />
|
||||
<div>
|
||||
<p className="text-xs text-gray-500">Stores</p>
|
||||
<p className="text-lg font-bold">{metrics.total_stores.toLocaleString()}</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div
|
||||
onClick={() => navigate('/admin/orchestrator/stores?status=healthy')}
|
||||
className="bg-white rounded-lg border border-green-200 p-4 cursor-pointer hover:bg-green-100 hover:border-green-400 transition-colors"
|
||||
>
|
||||
<div className="flex items-center gap-2">
|
||||
<CheckCircle className="w-5 h-5 text-green-500" />
|
||||
<div>
|
||||
<p className="text-xs text-gray-500">Healthy</p>
|
||||
<p className="text-lg font-bold text-green-600">{metrics.healthy_count}</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div
|
||||
onClick={() => navigate('/admin/orchestrator/stores?status=sandbox')}
|
||||
className="bg-white rounded-lg border border-yellow-200 p-4 cursor-pointer hover:bg-yellow-100 hover:border-yellow-400 transition-colors"
|
||||
>
|
||||
<div className="flex items-center gap-2">
|
||||
<Clock className="w-5 h-5 text-yellow-500" />
|
||||
<div>
|
||||
<p className="text-xs text-gray-500">Sandbox</p>
|
||||
<p className="text-lg font-bold text-yellow-600">{metrics.sandbox_count}</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div
|
||||
onClick={() => navigate('/admin/orchestrator/stores?status=needs_manual')}
|
||||
className="bg-white rounded-lg border border-orange-200 p-4 cursor-pointer hover:bg-orange-100 hover:border-orange-400 transition-colors"
|
||||
>
|
||||
<div className="flex items-center gap-2">
|
||||
<AlertTriangle className="w-5 h-5 text-orange-500" />
|
||||
<div>
|
||||
<p className="text-xs text-gray-500">Manual</p>
|
||||
<p className="text-lg font-bold text-orange-600">{metrics.needs_manual_count}</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div
|
||||
onClick={() => navigate('/admin/orchestrator/stores?status=failing')}
|
||||
className="bg-white rounded-lg border border-red-200 p-4 cursor-pointer hover:bg-red-100 hover:border-red-400 transition-colors"
|
||||
>
|
||||
<div className="flex items-center gap-2">
|
||||
<XCircle className="w-5 h-5 text-red-500" />
|
||||
<div>
|
||||
<p className="text-xs text-gray-500">Failing</p>
|
||||
<p className="text-lg font-bold text-red-600">{metrics.failing_count}</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* State Selector */}
|
||||
<div className="flex items-center gap-4">
|
||||
<label className="text-sm font-medium text-gray-700">Filter by State:</label>
|
||||
<div className="dropdown">
|
||||
<button tabIndex={0} className="btn btn-sm btn-outline gap-2">
|
||||
{selectedState === 'all' ? 'All States' : selectedState}
|
||||
<ChevronDown className="w-4 h-4" />
|
||||
</button>
|
||||
<ul tabIndex={0} className="dropdown-content z-[1] menu p-2 shadow bg-base-100 rounded-box w-52 max-h-60 overflow-y-auto">
|
||||
<li>
|
||||
<a onClick={() => setSelectedState('all')} className={selectedState === 'all' ? 'active' : ''}>
|
||||
All States ({states.reduce((sum, s) => sum + s.storeCount, 0)})
|
||||
</a>
|
||||
</li>
|
||||
<li className="divider"></li>
|
||||
{states.map((s) => (
|
||||
<li key={s.state}>
|
||||
<a onClick={() => setSelectedState(s.state)} className={selectedState === s.state ? 'active' : ''}>
|
||||
{s.state} ({s.storeCount})
|
||||
</a>
|
||||
</li>
|
||||
))}
|
||||
</ul>
|
||||
</div>
|
||||
<span className="text-sm text-gray-500">
|
||||
Showing {stores.length} of {totalStores} stores
|
||||
</span>
|
||||
</div>
|
||||
|
||||
{/* Two-column layout: Store table + Panel */}
|
||||
<div className="grid grid-cols-1 lg:grid-cols-3 gap-6">
|
||||
{/* Stores Table */}
|
||||
<div className="lg:col-span-2 bg-white rounded-lg border border-gray-200">
|
||||
<div className="p-4 border-b border-gray-200">
|
||||
<h2 className="text-lg font-semibold text-gray-900">Stores</h2>
|
||||
</div>
|
||||
<div className="overflow-x-auto max-h-[600px] overflow-y-auto">
|
||||
<table className="table table-sm w-full">
|
||||
<thead className="sticky top-0 bg-gray-50">
|
||||
<tr>
|
||||
<th>Name</th>
|
||||
<th>State</th>
|
||||
<th>Provider</th>
|
||||
<th>Status</th>
|
||||
<th>Last Success</th>
|
||||
<th>Last Failure</th>
|
||||
<th>Products</th>
|
||||
<th>Actions</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{stores.map((store) => (
|
||||
<tr
|
||||
key={store.id}
|
||||
className={`hover:bg-gray-50 cursor-pointer ${selectedStore?.id === store.id ? 'bg-blue-50' : ''}`}
|
||||
onClick={() => setSelectedStore(store)}
|
||||
>
|
||||
<td>
|
||||
<div className="font-medium text-gray-900">{store.name}</div>
|
||||
<div className="text-xs text-gray-500">{store.city}</div>
|
||||
</td>
|
||||
<td className="text-sm">{store.state}</td>
|
||||
<td>
|
||||
<span className="badge badge-sm badge-outline">{store.provider_display || 'Menu'}</span>
|
||||
</td>
|
||||
<td>{getStatusPill(store.status)}</td>
|
||||
<td className="text-xs text-green-600">
|
||||
{formatTimeAgo(store.lastSuccessAt)}
|
||||
</td>
|
||||
<td className="text-xs text-red-600">
|
||||
{formatTimeAgo(store.lastFailureAt)}
|
||||
</td>
|
||||
<td className="text-sm font-mono">
|
||||
{store.productCount.toLocaleString()}
|
||||
</td>
|
||||
<td>
|
||||
<div className="flex gap-1">
|
||||
<button
|
||||
className="btn btn-xs btn-outline btn-primary"
|
||||
title="Crawler Control"
|
||||
onClick={(e) => {
|
||||
e.stopPropagation();
|
||||
setSelectedStore(store);
|
||||
setPanelTab('control');
|
||||
}}
|
||||
>
|
||||
<TrendingUp className="w-3 h-3" />
|
||||
</button>
|
||||
<button
|
||||
className="btn btn-xs btn-outline btn-info"
|
||||
title="View Trace"
|
||||
onClick={(e) => {
|
||||
e.stopPropagation();
|
||||
setSelectedStore(store);
|
||||
setPanelTab('trace');
|
||||
}}
|
||||
>
|
||||
<FileText className="w-3 h-3" />
|
||||
</button>
|
||||
<button
|
||||
className="btn btn-xs btn-outline"
|
||||
title="View Profile"
|
||||
onClick={(e) => {
|
||||
e.stopPropagation();
|
||||
setSelectedStore(store);
|
||||
setPanelTab('profile');
|
||||
}}
|
||||
>
|
||||
<Settings className="w-3 h-3" />
|
||||
</button>
|
||||
<button
|
||||
className="btn btn-xs btn-outline"
|
||||
title="View Crawler Module"
|
||||
onClick={(e) => {
|
||||
e.stopPropagation();
|
||||
setSelectedStore(store);
|
||||
setPanelTab('module');
|
||||
}}
|
||||
>
|
||||
<Code className="w-3 h-3" />
|
||||
</button>
|
||||
</div>
|
||||
</td>
|
||||
</tr>
|
||||
))}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Side Panel */}
|
||||
<div className="lg:col-span-1">
|
||||
{selectedStore ? (
|
||||
<StoreOrchestratorPanel
|
||||
store={selectedStore}
|
||||
activeTab={panelTab}
|
||||
onTabChange={setPanelTab}
|
||||
onClose={() => setSelectedStore(null)}
|
||||
/>
|
||||
) : (
|
||||
<div className="bg-white rounded-lg border border-gray-200 p-6 text-center">
|
||||
<div className="text-gray-400 mb-2">
|
||||
<FileText className="w-12 h-12 mx-auto" />
|
||||
</div>
|
||||
<p className="text-gray-600">Select a store to view details</p>
|
||||
<p className="text-sm text-gray-500 mt-1">
|
||||
Click on any row or use the action buttons
|
||||
</p>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</Layout>
|
||||
);
|
||||
}
|
||||
264
cannaiq/src/pages/OrchestratorStores.tsx
Normal file
264
cannaiq/src/pages/OrchestratorStores.tsx
Normal file
@@ -0,0 +1,264 @@
|
||||
import { useEffect, useState } from 'react';
|
||||
import { useSearchParams, useNavigate } from 'react-router-dom';
|
||||
import { Layout } from '../components/Layout';
|
||||
import { api } from '../lib/api';
|
||||
import {
|
||||
Building2,
|
||||
ArrowLeft,
|
||||
CheckCircle,
|
||||
Clock,
|
||||
AlertTriangle,
|
||||
XCircle,
|
||||
RefreshCw,
|
||||
} from 'lucide-react';
|
||||
|
||||
interface StoreInfo {
|
||||
id: number;
|
||||
name: string;
|
||||
city: string;
|
||||
state: string;
|
||||
provider: string;
|
||||
provider_display?: string;
|
||||
platformDispensaryId: string | null;
|
||||
status: string;
|
||||
profileId: number | null;
|
||||
profileKey: string | null;
|
||||
lastCrawlAt: string | null;
|
||||
lastSuccessAt: string | null;
|
||||
lastFailureAt: string | null;
|
||||
productCount: number;
|
||||
}
|
||||
|
||||
const STATUS_FILTERS: Record<string, { label: string; match: (s: string) => boolean; icon: React.ReactNode; color: string }> = {
|
||||
all: {
|
||||
label: 'All Stores',
|
||||
match: () => true,
|
||||
icon: <Building2 className="w-4 h-4" />,
|
||||
color: 'text-gray-600',
|
||||
},
|
||||
healthy: {
|
||||
label: 'Healthy',
|
||||
match: (s) => s === 'production',
|
||||
icon: <CheckCircle className="w-4 h-4" />,
|
||||
color: 'text-green-600',
|
||||
},
|
||||
sandbox: {
|
||||
label: 'Sandbox',
|
||||
match: (s) => s === 'sandbox',
|
||||
icon: <Clock className="w-4 h-4" />,
|
||||
color: 'text-yellow-600',
|
||||
},
|
||||
needs_manual: {
|
||||
label: 'Needs Manual',
|
||||
match: (s) => s === 'needs_manual',
|
||||
icon: <AlertTriangle className="w-4 h-4" />,
|
||||
color: 'text-orange-600',
|
||||
},
|
||||
failing: {
|
||||
label: 'Failing',
|
||||
match: (s) => s === 'failing' || s === 'disabled',
|
||||
icon: <XCircle className="w-4 h-4" />,
|
||||
color: 'text-red-600',
|
||||
},
|
||||
};
|
||||
|
||||
export function OrchestratorStores() {
|
||||
const [searchParams, setSearchParams] = useSearchParams();
|
||||
const navigate = useNavigate();
|
||||
const [stores, setStores] = useState<StoreInfo[]>([]);
|
||||
const [loading, setLoading] = useState(true);
|
||||
const [totalStores, setTotalStores] = useState(0);
|
||||
|
||||
const statusFilter = searchParams.get('status') || 'all';
|
||||
|
||||
useEffect(() => {
|
||||
loadStores();
|
||||
}, []);
|
||||
|
||||
const loadStores = async () => {
|
||||
try {
|
||||
setLoading(true);
|
||||
const data = await api.getOrchestratorStores({ limit: 500 });
|
||||
setStores(data.stores || []);
|
||||
setTotalStores(data.total || 0);
|
||||
} catch (error) {
|
||||
console.error('Failed to load stores:', error);
|
||||
} finally {
|
||||
setLoading(false);
|
||||
}
|
||||
};
|
||||
|
||||
const filteredStores = stores.filter((store) => {
|
||||
const filter = STATUS_FILTERS[statusFilter];
|
||||
return filter ? filter.match(store.status) : true;
|
||||
});
|
||||
|
||||
const getStatusPill = (status: string) => {
|
||||
switch (status) {
|
||||
case 'production':
|
||||
return (
|
||||
<span className="badge badge-success badge-sm gap-1">
|
||||
<CheckCircle className="w-3 h-3" />
|
||||
PRODUCTION
|
||||
</span>
|
||||
);
|
||||
case 'sandbox':
|
||||
return (
|
||||
<span className="badge badge-warning badge-sm gap-1">
|
||||
<Clock className="w-3 h-3" />
|
||||
SANDBOX
|
||||
</span>
|
||||
);
|
||||
case 'needs_manual':
|
||||
return (
|
||||
<span className="badge badge-error badge-sm gap-1">
|
||||
<AlertTriangle className="w-3 h-3" />
|
||||
NEEDS MANUAL
|
||||
</span>
|
||||
);
|
||||
case 'disabled':
|
||||
case 'failing':
|
||||
return (
|
||||
<span className="badge badge-ghost badge-sm gap-1">
|
||||
<XCircle className="w-3 h-3" />
|
||||
{status.toUpperCase()}
|
||||
</span>
|
||||
);
|
||||
default:
|
||||
return (
|
||||
<span className="badge badge-outline badge-sm">
|
||||
{status || 'LEGACY'}
|
||||
</span>
|
||||
);
|
||||
}
|
||||
};
|
||||
|
||||
const formatTimeAgo = (dateStr: string | null) => {
|
||||
if (!dateStr) return '-';
|
||||
const date = new Date(dateStr);
|
||||
const now = new Date();
|
||||
const diff = now.getTime() - date.getTime();
|
||||
const hours = Math.floor(diff / (1000 * 60 * 60));
|
||||
const days = Math.floor(hours / 24);
|
||||
if (days > 0) return `${days}d ago`;
|
||||
if (hours > 0) return `${hours}h ago`;
|
||||
const minutes = Math.floor(diff / (1000 * 60));
|
||||
if (minutes > 0) return `${minutes}m ago`;
|
||||
return 'just now';
|
||||
};
|
||||
|
||||
return (
|
||||
<Layout>
|
||||
<div className="space-y-6">
|
||||
{/* Header */}
|
||||
<div className="flex items-center justify-between">
|
||||
<div className="flex items-center gap-4">
|
||||
<button
|
||||
onClick={() => navigate('/admin/orchestrator')}
|
||||
className="btn btn-sm btn-ghost gap-2"
|
||||
>
|
||||
<ArrowLeft className="w-4 h-4" />
|
||||
Back
|
||||
</button>
|
||||
<div>
|
||||
<h1 className="text-2xl font-bold text-gray-900">
|
||||
Stores
|
||||
{statusFilter !== 'all' && (
|
||||
<span className={`ml-2 text-lg ${STATUS_FILTERS[statusFilter]?.color}`}>
|
||||
({STATUS_FILTERS[statusFilter]?.label})
|
||||
</span>
|
||||
)}
|
||||
</h1>
|
||||
<p className="text-sm text-gray-600">
|
||||
{filteredStores.length} of {totalStores} stores
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
<button
|
||||
onClick={loadStores}
|
||||
className="btn btn-sm btn-outline gap-2"
|
||||
disabled={loading}
|
||||
>
|
||||
<RefreshCw className={`w-4 h-4 ${loading ? 'animate-spin' : ''}`} />
|
||||
Refresh
|
||||
</button>
|
||||
</div>
|
||||
|
||||
{/* Status Filter Tabs */}
|
||||
<div className="flex gap-2 flex-wrap">
|
||||
{Object.entries(STATUS_FILTERS).map(([key, { label, icon, color }]) => (
|
||||
<button
|
||||
key={key}
|
||||
onClick={() => setSearchParams(key === 'all' ? {} : { status: key })}
|
||||
className={`btn btn-sm gap-2 ${
|
||||
statusFilter === key ? 'btn-primary' : 'btn-outline'
|
||||
}`}
|
||||
>
|
||||
<span className={statusFilter === key ? 'text-white' : color}>{icon}</span>
|
||||
{label}
|
||||
<span className="badge badge-sm">
|
||||
{stores.filter((s) => STATUS_FILTERS[key].match(s.status)).length}
|
||||
</span>
|
||||
</button>
|
||||
))}
|
||||
</div>
|
||||
|
||||
{/* Stores Table */}
|
||||
<div className="bg-white rounded-lg border border-gray-200">
|
||||
<div className="overflow-x-auto">
|
||||
<table className="table table-sm w-full">
|
||||
<thead className="bg-gray-50">
|
||||
<tr>
|
||||
<th>Name</th>
|
||||
<th>City</th>
|
||||
<th>State</th>
|
||||
<th>Provider</th>
|
||||
<th>Status</th>
|
||||
<th>Last Success</th>
|
||||
<th>Products</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{loading ? (
|
||||
<tr>
|
||||
<td colSpan={7} className="text-center py-8">
|
||||
<div className="inline-block animate-spin rounded-full h-6 w-6 border-2 border-blue-500 border-t-transparent"></div>
|
||||
</td>
|
||||
</tr>
|
||||
) : filteredStores.length === 0 ? (
|
||||
<tr>
|
||||
<td colSpan={7} className="text-center py-8 text-gray-500">
|
||||
No stores match this filter
|
||||
</td>
|
||||
</tr>
|
||||
) : (
|
||||
filteredStores.map((store) => (
|
||||
<tr
|
||||
key={store.id}
|
||||
className="hover:bg-gray-50 cursor-pointer"
|
||||
onClick={() => navigate(`/admin/orchestrator?store=${store.id}`)}
|
||||
>
|
||||
<td className="font-medium">{store.name}</td>
|
||||
<td>{store.city}</td>
|
||||
<td>{store.state}</td>
|
||||
<td>
|
||||
<span className="badge badge-sm badge-outline">
|
||||
{store.provider_display || store.provider || 'Menu'}
|
||||
</span>
|
||||
</td>
|
||||
<td>{getStatusPill(store.status)}</td>
|
||||
<td className="text-xs text-green-600">
|
||||
{formatTimeAgo(store.lastSuccessAt)}
|
||||
</td>
|
||||
<td className="font-mono">{store.productCount.toLocaleString()}</td>
|
||||
</tr>
|
||||
))
|
||||
)}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</Layout>
|
||||
);
|
||||
}
|
||||
485
cannaiq/src/pages/ScraperOverviewDashboard.tsx
Normal file
485
cannaiq/src/pages/ScraperOverviewDashboard.tsx
Normal file
@@ -0,0 +1,485 @@
|
||||
import { useState, useEffect, useCallback } from 'react';
|
||||
import { Layout } from '../components/Layout';
|
||||
import { WorkerRoleBadge } from '../components/WorkerRoleBadge';
|
||||
import { api } from '../lib/api';
|
||||
import {
|
||||
Package,
|
||||
Building2,
|
||||
Users,
|
||||
EyeOff,
|
||||
Eye,
|
||||
AlertTriangle,
|
||||
CheckCircle,
|
||||
XCircle,
|
||||
RefreshCw,
|
||||
Activity,
|
||||
TrendingUp,
|
||||
Clock,
|
||||
} from 'lucide-react';
|
||||
import {
|
||||
AreaChart,
|
||||
Area,
|
||||
BarChart,
|
||||
Bar,
|
||||
XAxis,
|
||||
YAxis,
|
||||
CartesianGrid,
|
||||
Tooltip,
|
||||
ResponsiveContainer,
|
||||
Legend,
|
||||
} from 'recharts';
|
||||
|
||||
interface ScraperOverviewData {
|
||||
kpi: {
|
||||
totalProducts: number;
|
||||
inStockProducts: number;
|
||||
totalDispensaries: number;
|
||||
crawlableDispensaries: number;
|
||||
visibilityLost24h: number;
|
||||
visibilityRestored24h: number;
|
||||
totalVisibilityLost: number;
|
||||
errors24h: number;
|
||||
successfulJobs24h: number;
|
||||
activeWorkers: number;
|
||||
};
|
||||
workers: Array<{
|
||||
worker_name: string;
|
||||
worker_role: string;
|
||||
enabled: boolean;
|
||||
last_status: string | null;
|
||||
last_run_at: string | null;
|
||||
next_run_at: string | null;
|
||||
}>;
|
||||
activityByHour: Array<{
|
||||
hour: string;
|
||||
successful: number;
|
||||
failed: number;
|
||||
total: number;
|
||||
}>;
|
||||
productGrowth: Array<{
|
||||
day: string;
|
||||
newProducts: number;
|
||||
}>;
|
||||
recentRuns: Array<{
|
||||
id: number;
|
||||
jobName: string;
|
||||
status: string;
|
||||
startedAt: string;
|
||||
completedAt: string | null;
|
||||
itemsProcessed: number;
|
||||
itemsSucceeded: number;
|
||||
itemsFailed: number;
|
||||
workerName: string | null;
|
||||
workerRole: string | null;
|
||||
visibilityLost: number;
|
||||
visibilityRestored: number;
|
||||
}>;
|
||||
visibilityChanges: Array<{
|
||||
dispensaryId: number;
|
||||
dispensaryName: string;
|
||||
state: string;
|
||||
lost24h: number;
|
||||
restored24h: number;
|
||||
latestLoss: string | null;
|
||||
latestRestore: string | null;
|
||||
}>;
|
||||
}
|
||||
|
||||
function formatRelativeTime(dateStr: string | null | undefined): string {
|
||||
if (!dateStr) return '-';
|
||||
const date = new Date(dateStr);
|
||||
const now = new Date();
|
||||
const diffMs = now.getTime() - date.getTime();
|
||||
const diffMins = Math.round(diffMs / 60000);
|
||||
|
||||
if (diffMins < 0) {
|
||||
const futureMins = Math.abs(diffMins);
|
||||
if (futureMins < 60) return `in ${futureMins}m`;
|
||||
if (futureMins < 1440) return `in ${Math.round(futureMins / 60)}h`;
|
||||
return `in ${Math.round(futureMins / 1440)}d`;
|
||||
}
|
||||
|
||||
if (diffMins < 1) return 'just now';
|
||||
if (diffMins < 60) return `${diffMins}m ago`;
|
||||
if (diffMins < 1440) return `${Math.round(diffMins / 60)}h ago`;
|
||||
return `${Math.round(diffMins / 1440)}d ago`;
|
||||
}
|
||||
|
||||
function formatHour(isoDate: string): string {
|
||||
const date = new Date(isoDate);
|
||||
return date.toLocaleTimeString('en-US', { hour: '2-digit', hour12: true });
|
||||
}
|
||||
|
||||
function formatDay(isoDate: string): string {
|
||||
const date = new Date(isoDate);
|
||||
return date.toLocaleDateString('en-US', { weekday: 'short', month: 'short', day: 'numeric' });
|
||||
}
|
||||
|
||||
function StatusBadge({ status }: { status: string | null }) {
|
||||
if (!status) return <span className="text-gray-400">-</span>;
|
||||
|
||||
const config: Record<string, { bg: string; text: string; icon: any }> = {
|
||||
success: { bg: 'bg-green-100', text: 'text-green-700', icon: CheckCircle },
|
||||
running: { bg: 'bg-blue-100', text: 'text-blue-700', icon: Activity },
|
||||
pending: { bg: 'bg-yellow-100', text: 'text-yellow-700', icon: Clock },
|
||||
error: { bg: 'bg-red-100', text: 'text-red-700', icon: XCircle },
|
||||
partial: { bg: 'bg-orange-100', text: 'text-orange-700', icon: AlertTriangle },
|
||||
};
|
||||
|
||||
const cfg = config[status] || { bg: 'bg-gray-100', text: 'text-gray-700', icon: Clock };
|
||||
const Icon = cfg.icon;
|
||||
|
||||
return (
|
||||
<span className={`inline-flex items-center gap-1 px-2 py-0.5 rounded-full text-xs font-medium ${cfg.bg} ${cfg.text}`}>
|
||||
<Icon className="w-3 h-3" />
|
||||
{status}
|
||||
</span>
|
||||
);
|
||||
}
|
||||
|
||||
interface KPICardProps {
|
||||
title: string;
|
||||
value: number | string;
|
||||
subtitle?: string;
|
||||
icon: any;
|
||||
iconBg: string;
|
||||
iconColor: string;
|
||||
trend?: 'up' | 'down' | 'neutral';
|
||||
trendValue?: string;
|
||||
}
|
||||
|
||||
function KPICard({ title, value, subtitle, icon: Icon, iconBg, iconColor, trend, trendValue }: KPICardProps) {
|
||||
return (
|
||||
<div className="bg-white rounded-lg border border-gray-200 p-5 hover:shadow-md transition-shadow">
|
||||
<div className="flex items-start justify-between">
|
||||
<div className="flex-1">
|
||||
<p className="text-sm font-medium text-gray-500">{title}</p>
|
||||
<p className="text-2xl font-bold text-gray-900 mt-1">
|
||||
{typeof value === 'number' ? value.toLocaleString() : value}
|
||||
</p>
|
||||
{subtitle && (
|
||||
<p className="text-xs text-gray-500 mt-1">{subtitle}</p>
|
||||
)}
|
||||
{trend && trendValue && (
|
||||
<p className={`text-xs mt-1 flex items-center gap-1 ${
|
||||
trend === 'up' ? 'text-green-600' : trend === 'down' ? 'text-red-600' : 'text-gray-500'
|
||||
}`}>
|
||||
<TrendingUp className={`w-3 h-3 ${trend === 'down' ? 'rotate-180' : ''}`} />
|
||||
{trendValue}
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
<div className={`w-10 h-10 ${iconBg} rounded-lg flex items-center justify-center flex-shrink-0`}>
|
||||
<Icon className={`w-5 h-5 ${iconColor}`} />
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
export function ScraperOverviewDashboard() {
|
||||
const [data, setData] = useState<ScraperOverviewData | null>(null);
|
||||
const [loading, setLoading] = useState(true);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
const [autoRefresh, setAutoRefresh] = useState(true);
|
||||
|
||||
const fetchData = useCallback(async () => {
|
||||
try {
|
||||
const result = await api.getScraperOverview();
|
||||
setData(result);
|
||||
setError(null);
|
||||
} catch (err: any) {
|
||||
setError(err.message || 'Failed to fetch scraper overview');
|
||||
console.error('Scraper overview error:', err);
|
||||
} finally {
|
||||
setLoading(false);
|
||||
}
|
||||
}, []);
|
||||
|
||||
useEffect(() => {
|
||||
fetchData();
|
||||
if (autoRefresh) {
|
||||
const interval = setInterval(fetchData, 30000);
|
||||
return () => clearInterval(interval);
|
||||
}
|
||||
}, [fetchData, autoRefresh]);
|
||||
|
||||
if (loading) {
|
||||
return (
|
||||
<Layout>
|
||||
<div className="flex items-center justify-center h-64">
|
||||
<RefreshCw className="w-8 h-8 text-gray-400 animate-spin" />
|
||||
</div>
|
||||
</Layout>
|
||||
);
|
||||
}
|
||||
|
||||
const workerNames = data?.workers
|
||||
?.filter(w => w.worker_name)
|
||||
.map(w => w.worker_name)
|
||||
.slice(0, 4)
|
||||
.join(', ') || 'None active';
|
||||
|
||||
// Format activity data for chart
|
||||
const activityChartData = data?.activityByHour?.map(item => ({
|
||||
...item,
|
||||
hourLabel: formatHour(item.hour),
|
||||
})) || [];
|
||||
|
||||
// Format product growth data for chart
|
||||
const growthChartData = data?.productGrowth?.map(item => ({
|
||||
...item,
|
||||
dayLabel: formatDay(item.day),
|
||||
})) || [];
|
||||
|
||||
return (
|
||||
<Layout>
|
||||
<div className="space-y-6">
|
||||
{/* Header */}
|
||||
<div className="flex items-center justify-between">
|
||||
<div>
|
||||
<h1 className="text-2xl font-bold text-gray-900">Scraper Overview</h1>
|
||||
<p className="text-gray-500 mt-1">
|
||||
System health and crawler metrics dashboard
|
||||
</p>
|
||||
</div>
|
||||
<div className="flex items-center gap-4">
|
||||
<label className="flex items-center gap-2 text-sm text-gray-600">
|
||||
<input
|
||||
type="checkbox"
|
||||
checked={autoRefresh}
|
||||
onChange={(e) => setAutoRefresh(e.target.checked)}
|
||||
className="rounded border-gray-300"
|
||||
/>
|
||||
Auto-refresh (30s)
|
||||
</label>
|
||||
<button
|
||||
onClick={() => fetchData()}
|
||||
className="flex items-center gap-2 px-4 py-2 bg-emerald-600 text-white rounded-lg hover:bg-emerald-700 transition-colors"
|
||||
>
|
||||
<RefreshCw className="w-4 h-4" />
|
||||
Refresh
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{error && (
|
||||
<div className="bg-red-50 border border-red-200 rounded-lg p-4">
|
||||
<p className="text-red-700">{error}</p>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* KPI Cards - Top Row */}
|
||||
{data && (
|
||||
<div className="grid grid-cols-2 md:grid-cols-3 lg:grid-cols-6 gap-4">
|
||||
<KPICard
|
||||
title="Total Products"
|
||||
value={data.kpi.totalProducts}
|
||||
subtitle={`${data.kpi.inStockProducts.toLocaleString()} in stock`}
|
||||
icon={Package}
|
||||
iconBg="bg-blue-100"
|
||||
iconColor="text-blue-600"
|
||||
/>
|
||||
<KPICard
|
||||
title="Dispensaries"
|
||||
value={data.kpi.totalDispensaries}
|
||||
subtitle={`${data.kpi.crawlableDispensaries} crawlable`}
|
||||
icon={Building2}
|
||||
iconBg="bg-purple-100"
|
||||
iconColor="text-purple-600"
|
||||
/>
|
||||
<KPICard
|
||||
title="Active Workers"
|
||||
value={data.kpi.activeWorkers}
|
||||
subtitle={workerNames}
|
||||
icon={Users}
|
||||
iconBg="bg-emerald-100"
|
||||
iconColor="text-emerald-600"
|
||||
/>
|
||||
<KPICard
|
||||
title="Visibility Lost (24h)"
|
||||
value={data.kpi.visibilityLost24h}
|
||||
subtitle="Products lost menu visibility"
|
||||
icon={EyeOff}
|
||||
iconBg="bg-orange-100"
|
||||
iconColor="text-orange-600"
|
||||
/>
|
||||
<KPICard
|
||||
title="Visibility Restored (24h)"
|
||||
value={data.kpi.visibilityRestored24h}
|
||||
subtitle="Products regained visibility"
|
||||
icon={Eye}
|
||||
iconBg="bg-green-100"
|
||||
iconColor="text-green-600"
|
||||
/>
|
||||
<KPICard
|
||||
title="Errors (24h)"
|
||||
value={data.kpi.errors24h}
|
||||
subtitle={`${data.kpi.successfulJobs24h} successful jobs`}
|
||||
icon={AlertTriangle}
|
||||
iconBg="bg-red-100"
|
||||
iconColor="text-red-600"
|
||||
/>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Charts Row */}
|
||||
{data && (
|
||||
<div className="grid grid-cols-1 lg:grid-cols-2 gap-6">
|
||||
{/* Scrape Activity Chart */}
|
||||
<div className="bg-white rounded-lg border border-gray-200 p-6">
|
||||
<h3 className="text-lg font-semibold text-gray-900 mb-4">Scrape Activity (24h)</h3>
|
||||
<div className="h-64">
|
||||
<ResponsiveContainer width="100%" height="100%">
|
||||
<BarChart data={activityChartData}>
|
||||
<CartesianGrid strokeDasharray="3 3" />
|
||||
<XAxis dataKey="hourLabel" tick={{ fontSize: 11 }} />
|
||||
<YAxis tick={{ fontSize: 11 }} />
|
||||
<Tooltip />
|
||||
<Legend />
|
||||
<Bar dataKey="successful" name="Successful" fill="#10b981" stackId="a" />
|
||||
<Bar dataKey="failed" name="Failed" fill="#ef4444" stackId="a" />
|
||||
</BarChart>
|
||||
</ResponsiveContainer>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Product Growth Chart */}
|
||||
<div className="bg-white rounded-lg border border-gray-200 p-6">
|
||||
<h3 className="text-lg font-semibold text-gray-900 mb-4">Product Growth (7 days)</h3>
|
||||
<div className="h-64">
|
||||
<ResponsiveContainer width="100%" height="100%">
|
||||
<AreaChart data={growthChartData}>
|
||||
<CartesianGrid strokeDasharray="3 3" />
|
||||
<XAxis dataKey="dayLabel" tick={{ fontSize: 11 }} />
|
||||
<YAxis tick={{ fontSize: 11 }} />
|
||||
<Tooltip />
|
||||
<Area
|
||||
type="monotone"
|
||||
dataKey="newProducts"
|
||||
name="New Products"
|
||||
stroke="#8b5cf6"
|
||||
fill="#8b5cf6"
|
||||
fillOpacity={0.3}
|
||||
/>
|
||||
</AreaChart>
|
||||
</ResponsiveContainer>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Bottom Panels */}
|
||||
{data && (
|
||||
<div className="grid grid-cols-1 lg:grid-cols-2 gap-6">
|
||||
{/* Recent Worker Runs */}
|
||||
<div className="bg-white rounded-lg border border-gray-200 overflow-hidden">
|
||||
<div className="px-6 py-4 border-b border-gray-200">
|
||||
<h3 className="text-lg font-semibold text-gray-900">Recent Worker Runs</h3>
|
||||
</div>
|
||||
<div className="max-h-80 overflow-y-auto">
|
||||
<table className="w-full">
|
||||
<thead className="bg-gray-50 sticky top-0">
|
||||
<tr>
|
||||
<th className="px-4 py-2 text-left text-xs font-medium text-gray-500 uppercase">Worker</th>
|
||||
<th className="px-4 py-2 text-left text-xs font-medium text-gray-500 uppercase">Role</th>
|
||||
<th className="px-4 py-2 text-left text-xs font-medium text-gray-500 uppercase">Status</th>
|
||||
<th className="px-4 py-2 text-left text-xs font-medium text-gray-500 uppercase">When</th>
|
||||
<th className="px-4 py-2 text-right text-xs font-medium text-gray-500 uppercase">Stats</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody className="divide-y divide-gray-100">
|
||||
{data.recentRuns.map((run) => (
|
||||
<tr key={run.id} className="hover:bg-gray-50">
|
||||
<td className="px-4 py-2 text-sm font-medium text-gray-900">
|
||||
{run.workerName || run.jobName}
|
||||
</td>
|
||||
<td className="px-4 py-2">
|
||||
<WorkerRoleBadge role={run.workerRole} size="sm" />
|
||||
</td>
|
||||
<td className="px-4 py-2">
|
||||
<StatusBadge status={run.status} />
|
||||
</td>
|
||||
<td className="px-4 py-2 text-sm text-gray-500">
|
||||
{formatRelativeTime(run.startedAt)}
|
||||
</td>
|
||||
<td className="px-4 py-2 text-right text-xs">
|
||||
<span className="text-gray-600">{run.itemsProcessed}</span>
|
||||
{run.visibilityLost > 0 && (
|
||||
<span className="ml-2 text-orange-600">-{run.visibilityLost}</span>
|
||||
)}
|
||||
{run.visibilityRestored > 0 && (
|
||||
<span className="ml-2 text-green-600">+{run.visibilityRestored}</span>
|
||||
)}
|
||||
</td>
|
||||
</tr>
|
||||
))}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Recent Visibility Changes */}
|
||||
<div className="bg-white rounded-lg border border-gray-200 overflow-hidden">
|
||||
<div className="px-6 py-4 border-b border-gray-200">
|
||||
<h3 className="text-lg font-semibold text-gray-900">Recent Visibility Changes</h3>
|
||||
</div>
|
||||
<div className="max-h-80 overflow-y-auto">
|
||||
{data.visibilityChanges.length === 0 ? (
|
||||
<div className="px-6 py-8 text-center text-gray-500">
|
||||
No visibility changes in the last 24 hours
|
||||
</div>
|
||||
) : (
|
||||
<table className="w-full">
|
||||
<thead className="bg-gray-50 sticky top-0">
|
||||
<tr>
|
||||
<th className="px-4 py-2 text-left text-xs font-medium text-gray-500 uppercase">Store</th>
|
||||
<th className="px-4 py-2 text-left text-xs font-medium text-gray-500 uppercase">State</th>
|
||||
<th className="px-4 py-2 text-center text-xs font-medium text-gray-500 uppercase">Lost</th>
|
||||
<th className="px-4 py-2 text-center text-xs font-medium text-gray-500 uppercase">Restored</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody className="divide-y divide-gray-100">
|
||||
{data.visibilityChanges.map((change) => (
|
||||
<tr key={change.dispensaryId} className="hover:bg-gray-50">
|
||||
<td className="px-4 py-2 text-sm font-medium text-gray-900 max-w-xs truncate">
|
||||
{change.dispensaryName}
|
||||
</td>
|
||||
<td className="px-4 py-2 text-sm text-gray-500">
|
||||
{change.state}
|
||||
</td>
|
||||
<td className="px-4 py-2 text-center">
|
||||
{change.lost24h > 0 ? (
|
||||
<span className="inline-flex items-center gap-1 px-2 py-0.5 rounded-full text-xs font-medium bg-orange-100 text-orange-700">
|
||||
<EyeOff className="w-3 h-3" />
|
||||
{change.lost24h}
|
||||
</span>
|
||||
) : (
|
||||
<span className="text-gray-400">-</span>
|
||||
)}
|
||||
</td>
|
||||
<td className="px-4 py-2 text-center">
|
||||
{change.restored24h > 0 ? (
|
||||
<span className="inline-flex items-center gap-1 px-2 py-0.5 rounded-full text-xs font-medium bg-green-100 text-green-700">
|
||||
<Eye className="w-3 h-3" />
|
||||
{change.restored24h}
|
||||
</span>
|
||||
) : (
|
||||
<span className="text-gray-400">-</span>
|
||||
)}
|
||||
</td>
|
||||
</tr>
|
||||
))}
|
||||
</tbody>
|
||||
</table>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</Layout>
|
||||
);
|
||||
}
|
||||
|
||||
export default ScraperOverviewDashboard;
|
||||
498
cannaiq/src/pages/WorkersDashboard.tsx
Normal file
498
cannaiq/src/pages/WorkersDashboard.tsx
Normal file
@@ -0,0 +1,498 @@
|
||||
import { useState, useEffect, useCallback } from 'react';
|
||||
import { Layout } from '../components/Layout';
|
||||
import { WorkerRoleBadge, formatScope } from '../components/WorkerRoleBadge';
|
||||
import { api } from '../lib/api';
|
||||
import {
|
||||
Users,
|
||||
Play,
|
||||
Clock,
|
||||
CheckCircle,
|
||||
XCircle,
|
||||
AlertTriangle,
|
||||
RefreshCw,
|
||||
ChevronDown,
|
||||
ChevronUp,
|
||||
Activity,
|
||||
} from 'lucide-react';
|
||||
|
||||
interface Schedule {
|
||||
id: number;
|
||||
job_name: string;
|
||||
description: string;
|
||||
worker_name: string;
|
||||
worker_role: string;
|
||||
enabled: boolean;
|
||||
base_interval_minutes: number;
|
||||
jitter_minutes: number;
|
||||
next_run_at: string | null;
|
||||
last_run_at: string | null;
|
||||
last_status: string | null;
|
||||
job_config: any;
|
||||
}
|
||||
|
||||
interface RunLog {
|
||||
id: number;
|
||||
schedule_id: number;
|
||||
job_name: string;
|
||||
status: string;
|
||||
started_at: string;
|
||||
completed_at: string | null;
|
||||
items_processed: number;
|
||||
items_succeeded: number;
|
||||
items_failed: number;
|
||||
error_message: string | null;
|
||||
metadata: any;
|
||||
worker_name: string;
|
||||
run_role: string;
|
||||
duration_seconds?: number;
|
||||
}
|
||||
|
||||
interface MonitorSummary {
|
||||
running_scheduled_jobs: number;
|
||||
running_dispensary_crawl_jobs: number;
|
||||
successful_jobs_24h: number;
|
||||
failed_jobs_24h: number;
|
||||
successful_crawls_24h: number;
|
||||
failed_crawls_24h: number;
|
||||
products_found_24h: number;
|
||||
snapshots_created_24h: number;
|
||||
last_job_started: string | null;
|
||||
last_job_completed: string | null;
|
||||
nextRuns: Schedule[];
|
||||
}
|
||||
|
||||
function formatDuration(seconds: number | null | undefined): string {
|
||||
if (!seconds) return '-';
|
||||
if (seconds < 60) return `${Math.round(seconds)}s`;
|
||||
if (seconds < 3600) return `${Math.round(seconds / 60)}m`;
|
||||
return `${Math.round(seconds / 3600)}h ${Math.round((seconds % 3600) / 60)}m`;
|
||||
}
|
||||
|
||||
function formatRelativeTime(dateStr: string | null | undefined): string {
|
||||
if (!dateStr) return '-';
|
||||
const date = new Date(dateStr);
|
||||
const now = new Date();
|
||||
const diffMs = now.getTime() - date.getTime();
|
||||
const diffMins = Math.round(diffMs / 60000);
|
||||
|
||||
if (diffMins < 0) {
|
||||
const futureMins = Math.abs(diffMins);
|
||||
if (futureMins < 60) return `in ${futureMins}m`;
|
||||
if (futureMins < 1440) return `in ${Math.round(futureMins / 60)}h`;
|
||||
return `in ${Math.round(futureMins / 1440)}d`;
|
||||
}
|
||||
|
||||
if (diffMins < 1) return 'just now';
|
||||
if (diffMins < 60) return `${diffMins}m ago`;
|
||||
if (diffMins < 1440) return `${Math.round(diffMins / 60)}h ago`;
|
||||
return `${Math.round(diffMins / 1440)}d ago`;
|
||||
}
|
||||
|
||||
function StatusBadge({ status }: { status: string | null }) {
|
||||
if (!status) return <span className="text-gray-400">-</span>;
|
||||
|
||||
const config: Record<string, { bg: string; text: string; icon: any }> = {
|
||||
success: { bg: 'bg-green-100', text: 'text-green-700', icon: CheckCircle },
|
||||
running: { bg: 'bg-blue-100', text: 'text-blue-700', icon: Activity },
|
||||
pending: { bg: 'bg-yellow-100', text: 'text-yellow-700', icon: Clock },
|
||||
error: { bg: 'bg-red-100', text: 'text-red-700', icon: XCircle },
|
||||
partial: { bg: 'bg-orange-100', text: 'text-orange-700', icon: AlertTriangle },
|
||||
};
|
||||
|
||||
const cfg = config[status] || { bg: 'bg-gray-100', text: 'text-gray-700', icon: Clock };
|
||||
const Icon = cfg.icon;
|
||||
|
||||
return (
|
||||
<span className={`inline-flex items-center gap-1 px-2 py-0.5 rounded-full text-xs font-medium ${cfg.bg} ${cfg.text}`}>
|
||||
<Icon className="w-3 h-3" />
|
||||
{status}
|
||||
</span>
|
||||
);
|
||||
}
|
||||
|
||||
export function WorkersDashboard() {
|
||||
const [schedules, setSchedules] = useState<Schedule[]>([]);
|
||||
const [selectedWorker, setSelectedWorker] = useState<Schedule | null>(null);
|
||||
const [workerLogs, setWorkerLogs] = useState<RunLog[]>([]);
|
||||
const [summary, setSummary] = useState<MonitorSummary | null>(null);
|
||||
const [loading, setLoading] = useState(true);
|
||||
const [logsLoading, setLogsLoading] = useState(false);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
const [triggering, setTriggering] = useState<number | null>(null);
|
||||
|
||||
const fetchData = useCallback(async () => {
|
||||
try {
|
||||
const [schedulesRes, summaryRes] = await Promise.all([
|
||||
api.get('/api/dutchie-az/admin/schedules'),
|
||||
api.get('/api/dutchie-az/monitor/summary'),
|
||||
]);
|
||||
|
||||
setSchedules(schedulesRes.data.schedules || []);
|
||||
setSummary(summaryRes.data);
|
||||
setError(null);
|
||||
} catch (err: any) {
|
||||
setError(err.message || 'Failed to fetch data');
|
||||
} finally {
|
||||
setLoading(false);
|
||||
}
|
||||
}, []);
|
||||
|
||||
const fetchWorkerLogs = useCallback(async (scheduleId: number) => {
|
||||
setLogsLoading(true);
|
||||
try {
|
||||
const res = await api.get(`/api/dutchie-az/admin/schedules/${scheduleId}/logs?limit=20`);
|
||||
setWorkerLogs(res.data.logs || []);
|
||||
} catch (err: any) {
|
||||
console.error('Failed to fetch worker logs:', err);
|
||||
setWorkerLogs([]);
|
||||
} finally {
|
||||
setLogsLoading(false);
|
||||
}
|
||||
}, []);
|
||||
|
||||
useEffect(() => {
|
||||
fetchData();
|
||||
const interval = setInterval(fetchData, 30000);
|
||||
return () => clearInterval(interval);
|
||||
}, [fetchData]);
|
||||
|
||||
useEffect(() => {
|
||||
if (selectedWorker) {
|
||||
fetchWorkerLogs(selectedWorker.id);
|
||||
} else {
|
||||
setWorkerLogs([]);
|
||||
}
|
||||
}, [selectedWorker, fetchWorkerLogs]);
|
||||
|
||||
const handleSelectWorker = (schedule: Schedule) => {
|
||||
if (selectedWorker?.id === schedule.id) {
|
||||
setSelectedWorker(null);
|
||||
} else {
|
||||
setSelectedWorker(schedule);
|
||||
}
|
||||
};
|
||||
|
||||
const handleTrigger = async (scheduleId: number) => {
|
||||
setTriggering(scheduleId);
|
||||
try {
|
||||
await api.post(`/api/dutchie-az/admin/schedules/${scheduleId}/trigger`);
|
||||
// Refresh data after trigger
|
||||
setTimeout(fetchData, 1000);
|
||||
} catch (err: any) {
|
||||
console.error('Failed to trigger schedule:', err);
|
||||
} finally {
|
||||
setTriggering(null);
|
||||
}
|
||||
};
|
||||
|
||||
if (loading) {
|
||||
return (
|
||||
<Layout>
|
||||
<div className="flex items-center justify-center h-64">
|
||||
<RefreshCw className="w-8 h-8 text-gray-400 animate-spin" />
|
||||
</div>
|
||||
</Layout>
|
||||
);
|
||||
}
|
||||
|
||||
return (
|
||||
<Layout>
|
||||
<div className="space-y-6">
|
||||
{/* Header */}
|
||||
<div className="flex items-center justify-between">
|
||||
<div>
|
||||
<h1 className="text-2xl font-bold text-gray-900">Crawler Workers</h1>
|
||||
<p className="text-gray-500 mt-1">
|
||||
Named workforce dashboard - Alice, Henry, Bella, Oscar
|
||||
</p>
|
||||
</div>
|
||||
<button
|
||||
onClick={() => fetchData()}
|
||||
className="flex items-center gap-2 px-4 py-2 bg-emerald-600 text-white rounded-lg hover:bg-emerald-700 transition-colors"
|
||||
>
|
||||
<RefreshCw className="w-4 h-4" />
|
||||
Refresh
|
||||
</button>
|
||||
</div>
|
||||
|
||||
{error && (
|
||||
<div className="bg-red-50 border border-red-200 rounded-lg p-4">
|
||||
<p className="text-red-700">{error}</p>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Summary Cards */}
|
||||
{summary && (
|
||||
<div className="grid grid-cols-4 gap-4">
|
||||
<div className="bg-white rounded-lg border border-gray-200 p-4">
|
||||
<div className="flex items-center gap-3">
|
||||
<div className="w-10 h-10 bg-blue-100 rounded-lg flex items-center justify-center">
|
||||
<Activity className="w-5 h-5 text-blue-600" />
|
||||
</div>
|
||||
<div>
|
||||
<p className="text-sm text-gray-500">Running Jobs</p>
|
||||
<p className="text-xl font-semibold">
|
||||
{summary.running_scheduled_jobs + summary.running_dispensary_crawl_jobs}
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div className="bg-white rounded-lg border border-gray-200 p-4">
|
||||
<div className="flex items-center gap-3">
|
||||
<div className="w-10 h-10 bg-green-100 rounded-lg flex items-center justify-center">
|
||||
<CheckCircle className="w-5 h-5 text-green-600" />
|
||||
</div>
|
||||
<div>
|
||||
<p className="text-sm text-gray-500">Successful (24h)</p>
|
||||
<p className="text-xl font-semibold">{summary.successful_jobs_24h}</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div className="bg-white rounded-lg border border-gray-200 p-4">
|
||||
<div className="flex items-center gap-3">
|
||||
<div className="w-10 h-10 bg-red-100 rounded-lg flex items-center justify-center">
|
||||
<XCircle className="w-5 h-5 text-red-600" />
|
||||
</div>
|
||||
<div>
|
||||
<p className="text-sm text-gray-500">Failed (24h)</p>
|
||||
<p className="text-xl font-semibold">{summary.failed_jobs_24h}</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div className="bg-white rounded-lg border border-gray-200 p-4">
|
||||
<div className="flex items-center gap-3">
|
||||
<div className="w-10 h-10 bg-purple-100 rounded-lg flex items-center justify-center">
|
||||
<Users className="w-5 h-5 text-purple-600" />
|
||||
</div>
|
||||
<div>
|
||||
<p className="text-sm text-gray-500">Active Workers</p>
|
||||
<p className="text-xl font-semibold">{schedules.filter(s => s.enabled).length}</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Workers Table */}
|
||||
<div className="bg-white rounded-lg border border-gray-200 overflow-hidden">
|
||||
<div className="px-6 py-4 border-b border-gray-200">
|
||||
<h2 className="text-lg font-semibold text-gray-900">Workers</h2>
|
||||
</div>
|
||||
<table className="w-full">
|
||||
<thead className="bg-gray-50">
|
||||
<tr>
|
||||
<th className="px-6 py-3 text-left text-xs font-medium text-gray-500 uppercase tracking-wider">
|
||||
Worker
|
||||
</th>
|
||||
<th className="px-6 py-3 text-left text-xs font-medium text-gray-500 uppercase tracking-wider">
|
||||
Role
|
||||
</th>
|
||||
<th className="px-6 py-3 text-left text-xs font-medium text-gray-500 uppercase tracking-wider">
|
||||
Scope
|
||||
</th>
|
||||
<th className="px-6 py-3 text-left text-xs font-medium text-gray-500 uppercase tracking-wider">
|
||||
Last Run
|
||||
</th>
|
||||
<th className="px-6 py-3 text-left text-xs font-medium text-gray-500 uppercase tracking-wider">
|
||||
Next Run
|
||||
</th>
|
||||
<th className="px-6 py-3 text-left text-xs font-medium text-gray-500 uppercase tracking-wider">
|
||||
Status
|
||||
</th>
|
||||
<th className="px-6 py-3 text-right text-xs font-medium text-gray-500 uppercase tracking-wider">
|
||||
Actions
|
||||
</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody className="divide-y divide-gray-200">
|
||||
{schedules.map((schedule) => (
|
||||
<tr
|
||||
key={schedule.id}
|
||||
className={`hover:bg-gray-50 cursor-pointer transition-colors ${
|
||||
selectedWorker?.id === schedule.id ? 'bg-emerald-50' : ''
|
||||
}`}
|
||||
onClick={() => handleSelectWorker(schedule)}
|
||||
>
|
||||
<td className="px-6 py-4 whitespace-nowrap">
|
||||
<div className="flex items-center gap-2">
|
||||
<span
|
||||
className={`w-2 h-2 rounded-full ${
|
||||
schedule.enabled ? 'bg-green-500' : 'bg-gray-300'
|
||||
}`}
|
||||
/>
|
||||
<span className="font-medium text-gray-900">
|
||||
{schedule.worker_name || schedule.job_name}
|
||||
</span>
|
||||
{selectedWorker?.id === schedule.id ? (
|
||||
<ChevronUp className="w-4 h-4 text-gray-400" />
|
||||
) : (
|
||||
<ChevronDown className="w-4 h-4 text-gray-400" />
|
||||
)}
|
||||
</div>
|
||||
{schedule.description && (
|
||||
<p className="text-xs text-gray-500 mt-0.5">{schedule.description}</p>
|
||||
)}
|
||||
</td>
|
||||
<td className="px-6 py-4 whitespace-nowrap">
|
||||
<WorkerRoleBadge role={schedule.worker_role} size="md" />
|
||||
</td>
|
||||
<td className="px-6 py-4 whitespace-nowrap text-sm text-gray-500">
|
||||
{formatScope(schedule.job_config)}
|
||||
</td>
|
||||
<td className="px-6 py-4 whitespace-nowrap text-sm text-gray-500">
|
||||
{formatRelativeTime(schedule.last_run_at)}
|
||||
</td>
|
||||
<td className="px-6 py-4 whitespace-nowrap text-sm text-gray-500">
|
||||
{schedule.enabled ? formatRelativeTime(schedule.next_run_at) : 'disabled'}
|
||||
</td>
|
||||
<td className="px-6 py-4 whitespace-nowrap">
|
||||
<StatusBadge status={schedule.last_status} />
|
||||
</td>
|
||||
<td className="px-6 py-4 whitespace-nowrap text-right">
|
||||
<button
|
||||
onClick={(e) => {
|
||||
e.stopPropagation();
|
||||
handleTrigger(schedule.id);
|
||||
}}
|
||||
disabled={triggering === schedule.id}
|
||||
className="inline-flex items-center gap-1 px-3 py-1.5 bg-emerald-100 text-emerald-700 rounded-lg text-sm font-medium hover:bg-emerald-200 transition-colors disabled:opacity-50"
|
||||
>
|
||||
{triggering === schedule.id ? (
|
||||
<RefreshCw className="w-3.5 h-3.5 animate-spin" />
|
||||
) : (
|
||||
<Play className="w-3.5 h-3.5" />
|
||||
)}
|
||||
Run Now
|
||||
</button>
|
||||
</td>
|
||||
</tr>
|
||||
))}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
|
||||
{/* Worker Detail Pane */}
|
||||
{selectedWorker && (
|
||||
<div className="bg-white rounded-lg border border-gray-200 overflow-hidden">
|
||||
<div className="px-6 py-4 border-b border-gray-200 bg-gray-50">
|
||||
<div className="flex items-center justify-between">
|
||||
<div className="flex items-center gap-3">
|
||||
<h2 className="text-lg font-semibold text-gray-900">
|
||||
{selectedWorker.worker_name || selectedWorker.job_name}
|
||||
</h2>
|
||||
<WorkerRoleBadge role={selectedWorker.worker_role} size="md" />
|
||||
</div>
|
||||
<div className="text-sm text-gray-500">
|
||||
Scope: {formatScope(selectedWorker.job_config)}
|
||||
</div>
|
||||
</div>
|
||||
{selectedWorker.description && (
|
||||
<p className="text-sm text-gray-500 mt-1">{selectedWorker.description}</p>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{/* Run History */}
|
||||
<div className="px-6 py-4">
|
||||
<h3 className="text-sm font-medium text-gray-700 mb-3">Recent Run History</h3>
|
||||
{logsLoading ? (
|
||||
<div className="flex items-center justify-center py-8">
|
||||
<RefreshCw className="w-6 h-6 text-gray-400 animate-spin" />
|
||||
</div>
|
||||
) : workerLogs.length === 0 ? (
|
||||
<p className="text-sm text-gray-500 py-4 text-center">No run history available</p>
|
||||
) : (
|
||||
<table className="w-full">
|
||||
<thead className="bg-gray-50">
|
||||
<tr>
|
||||
<th className="px-4 py-2 text-left text-xs font-medium text-gray-500 uppercase">
|
||||
Started
|
||||
</th>
|
||||
<th className="px-4 py-2 text-left text-xs font-medium text-gray-500 uppercase">
|
||||
Duration
|
||||
</th>
|
||||
<th className="px-4 py-2 text-left text-xs font-medium text-gray-500 uppercase">
|
||||
Status
|
||||
</th>
|
||||
<th className="px-4 py-2 text-left text-xs font-medium text-gray-500 uppercase">
|
||||
Processed
|
||||
</th>
|
||||
<th className="px-4 py-2 text-left text-xs font-medium text-gray-500 uppercase">
|
||||
Visibility Stats
|
||||
</th>
|
||||
<th className="px-4 py-2 text-left text-xs font-medium text-gray-500 uppercase">
|
||||
Error
|
||||
</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody className="divide-y divide-gray-100">
|
||||
{workerLogs.map((log) => {
|
||||
const duration = log.completed_at
|
||||
? (new Date(log.completed_at).getTime() -
|
||||
new Date(log.started_at).getTime()) /
|
||||
1000
|
||||
: null;
|
||||
const visLost = log.metadata?.visibilityLostCount;
|
||||
const visRestored = log.metadata?.visibilityRestoredCount;
|
||||
|
||||
return (
|
||||
<tr key={log.id} className="hover:bg-gray-50">
|
||||
<td className="px-4 py-2 text-sm text-gray-900">
|
||||
{formatRelativeTime(log.started_at)}
|
||||
</td>
|
||||
<td className="px-4 py-2 text-sm text-gray-500">
|
||||
{formatDuration(duration)}
|
||||
</td>
|
||||
<td className="px-4 py-2">
|
||||
<StatusBadge status={log.status} />
|
||||
</td>
|
||||
<td className="px-4 py-2 text-sm">
|
||||
<span className="text-green-600">{log.items_succeeded}</span>
|
||||
<span className="text-gray-400"> / </span>
|
||||
<span className="text-gray-600">{log.items_processed}</span>
|
||||
{log.items_failed > 0 && (
|
||||
<>
|
||||
<span className="text-gray-400"> (</span>
|
||||
<span className="text-red-600">{log.items_failed} failed</span>
|
||||
<span className="text-gray-400">)</span>
|
||||
</>
|
||||
)}
|
||||
</td>
|
||||
<td className="px-4 py-2 text-sm">
|
||||
{visLost !== undefined || visRestored !== undefined ? (
|
||||
<span className="flex items-center gap-2">
|
||||
{visLost !== undefined && visLost > 0 && (
|
||||
<span className="text-orange-600">
|
||||
-{visLost} lost
|
||||
</span>
|
||||
)}
|
||||
{visRestored !== undefined && visRestored > 0 && (
|
||||
<span className="text-green-600">
|
||||
+{visRestored} restored
|
||||
</span>
|
||||
)}
|
||||
{visLost === 0 && visRestored === 0 && (
|
||||
<span className="text-gray-400">no changes</span>
|
||||
)}
|
||||
</span>
|
||||
) : (
|
||||
<span className="text-gray-400">-</span>
|
||||
)}
|
||||
</td>
|
||||
<td className="px-4 py-2 text-sm text-red-600 max-w-xs truncate">
|
||||
{log.error_message || '-'}
|
||||
</td>
|
||||
</tr>
|
||||
);
|
||||
})}
|
||||
</tbody>
|
||||
</table>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</Layout>
|
||||
);
|
||||
}
|
||||
|
||||
export default WorkersDashboard;
|
||||
Reference in New Issue
Block a user