feat(cannaiq): Add Workers Dashboard and visibility tracking

Workers Dashboard:
- New /workers route with two-pane layout
- Workers table showing Alice, Henry, Bella, Oscar with role badges
- Run history with visibility stats (lost/restored counts)
- "Run Now" action to trigger workers immediately

Migrations:
- 057: Add visibility tracking columns (visibility_lost, visibility_lost_at, visibility_restored_at)
- 058: Add ID resolution columns for Henry worker
- 059: Add job queue columns (max_retries, retry_count, worker_id, locked_at, locked_by)

Backend fixes:
- Add httpStatus to CrawlResult interface for error classification
- Fix pool.ts typing for event listener
- Update completeJob to accept visibility stats in metadata

Frontend fixes:
- Fix NationalDashboard crash with safe formatMoney helper
- Fix OrchestratorDashboard/Stores StoreInfo type mismatches
- Add workerName/workerRole to getDutchieAZSchedules API type

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
Kelly
2025-12-07 11:04:12 -07:00
parent 1d1263afc6
commit 8ac64ba077
22 changed files with 7022 additions and 134 deletions

View File

@@ -0,0 +1,64 @@
-- Migration 057: Add visibility tracking columns to dutchie_products
--
-- Supports Bella (Product Sync) worker visibility-loss tracking:
-- - visibility_lost: TRUE when product disappears from GraphQL feed
-- - visibility_lost_at: Timestamp when product first went missing
-- - visibility_restored_at: Timestamp when product reappeared
--
-- These columns enable tracking of products that temporarily or permanently
-- disappear from Dutchie GraphQL API responses.
-- ============================================================
-- 1. ADD VISIBILITY TRACKING COLUMNS TO dutchie_products
-- ============================================================
ALTER TABLE dutchie_products
ADD COLUMN IF NOT EXISTS visibility_lost BOOLEAN DEFAULT FALSE,
ADD COLUMN IF NOT EXISTS visibility_lost_at TIMESTAMPTZ,
ADD COLUMN IF NOT EXISTS visibility_restored_at TIMESTAMPTZ;
COMMENT ON COLUMN dutchie_products.visibility_lost IS 'TRUE when product is missing from GraphQL feed';
COMMENT ON COLUMN dutchie_products.visibility_lost_at IS 'Timestamp when product first went missing from feed';
COMMENT ON COLUMN dutchie_products.visibility_restored_at IS 'Timestamp when product reappeared after being missing';
-- ============================================================
-- 2. CREATE INDEX FOR VISIBILITY QUERIES
-- ============================================================
CREATE INDEX IF NOT EXISTS idx_dutchie_products_visibility_lost
ON dutchie_products(visibility_lost)
WHERE visibility_lost = TRUE;
CREATE INDEX IF NOT EXISTS idx_dutchie_products_visibility_lost_at
ON dutchie_products(visibility_lost_at)
WHERE visibility_lost_at IS NOT NULL;
-- ============================================================
-- 3. CREATE VIEW FOR VISIBILITY ANALYTICS
-- ============================================================
CREATE OR REPLACE VIEW v_visibility_summary AS
SELECT
d.id AS dispensary_id,
d.name AS dispensary_name,
d.state,
COUNT(dp.id) AS total_products,
COUNT(dp.id) FILTER (WHERE dp.visibility_lost = TRUE) AS visibility_lost_count,
COUNT(dp.id) FILTER (WHERE dp.visibility_lost = FALSE OR dp.visibility_lost IS NULL) AS visible_count,
COUNT(dp.id) FILTER (WHERE dp.visibility_restored_at IS NOT NULL) AS restored_count,
MAX(dp.visibility_lost_at) AS latest_loss_at,
MAX(dp.visibility_restored_at) AS latest_restore_at
FROM dispensaries d
LEFT JOIN dutchie_products dp ON d.id = dp.dispensary_id
WHERE d.menu_type = 'dutchie'
GROUP BY d.id, d.name, d.state;
COMMENT ON VIEW v_visibility_summary IS 'Aggregated visibility metrics per dispensary for dashboard analytics';
-- ============================================================
-- 4. RECORD MIGRATION
-- ============================================================
INSERT INTO schema_migrations (version, name, applied_at)
VALUES (57, '057_visibility_tracking_columns', NOW())
ON CONFLICT (version) DO NOTHING;

View File

@@ -0,0 +1,46 @@
-- Migration 058: Add ID resolution tracking columns to dispensaries
--
-- Supports Henry (Entry Point Finder) worker tracking:
-- - id_resolution_attempts: Count of how many times we've tried to resolve platform ID
-- - last_id_resolution_at: When we last tried (matches code expectation)
-- - id_resolution_status: Current status (pending, resolved, failed)
-- - id_resolution_error: Last error message from resolution attempt
-- ============================================================
-- 1. ADD ID RESOLUTION COLUMNS TO dispensaries
-- ============================================================
ALTER TABLE dispensaries
ADD COLUMN IF NOT EXISTS id_resolution_attempts INTEGER DEFAULT 0,
ADD COLUMN IF NOT EXISTS last_id_resolution_at TIMESTAMPTZ,
ADD COLUMN IF NOT EXISTS id_resolution_status VARCHAR(20) DEFAULT 'pending',
ADD COLUMN IF NOT EXISTS id_resolution_error TEXT;
COMMENT ON COLUMN dispensaries.id_resolution_attempts IS 'Number of attempts to resolve platform_dispensary_id';
COMMENT ON COLUMN dispensaries.last_id_resolution_at IS 'Timestamp of last ID resolution attempt';
COMMENT ON COLUMN dispensaries.id_resolution_status IS 'Status: pending, resolved, failed';
COMMENT ON COLUMN dispensaries.id_resolution_error IS 'Last error message from ID resolution attempt';
-- Additional columns needed by worker/scheduler
ALTER TABLE dispensaries
ADD COLUMN IF NOT EXISTS failed_at TIMESTAMPTZ,
ADD COLUMN IF NOT EXISTS failure_notes TEXT;
COMMENT ON COLUMN dispensaries.failed_at IS 'Timestamp when dispensary was marked as permanently failed';
COMMENT ON COLUMN dispensaries.failure_notes IS 'Notes about why dispensary was marked as failed';
-- ============================================================
-- 2. CREATE INDEX FOR RESOLUTION QUERIES
-- ============================================================
CREATE INDEX IF NOT EXISTS idx_dispensaries_id_resolution_status
ON dispensaries(id_resolution_status)
WHERE id_resolution_status = 'pending';
-- ============================================================
-- 3. RECORD MIGRATION
-- ============================================================
INSERT INTO schema_migrations (version, name, applied_at)
VALUES (58, '058_add_id_resolution_columns', NOW())
ON CONFLICT (version) DO NOTHING;

View File

@@ -0,0 +1,67 @@
-- Migration 059: Add missing columns to dispensary_crawl_jobs
--
-- Required for worker job processing:
-- - max_retries: Maximum retry attempts for a job
-- - retry_count: Current retry count
-- - worker_id: ID of worker processing the job
-- - locked_at: When the job was locked by a worker
-- - locked_by: Hostname of worker that locked the job
-- ============================================================
-- 1. ADD JOB QUEUE COLUMNS
-- ============================================================
ALTER TABLE dispensary_crawl_jobs
ADD COLUMN IF NOT EXISTS max_retries INTEGER DEFAULT 3,
ADD COLUMN IF NOT EXISTS retry_count INTEGER DEFAULT 0,
ADD COLUMN IF NOT EXISTS worker_id VARCHAR(100),
ADD COLUMN IF NOT EXISTS locked_at TIMESTAMPTZ,
ADD COLUMN IF NOT EXISTS locked_by VARCHAR(100),
ADD COLUMN IF NOT EXISTS updated_at TIMESTAMPTZ DEFAULT NOW();
COMMENT ON COLUMN dispensary_crawl_jobs.max_retries IS 'Maximum number of retry attempts';
COMMENT ON COLUMN dispensary_crawl_jobs.retry_count IS 'Current retry count';
COMMENT ON COLUMN dispensary_crawl_jobs.worker_id IS 'ID of worker processing this job';
COMMENT ON COLUMN dispensary_crawl_jobs.locked_at IS 'When job was locked by worker';
COMMENT ON COLUMN dispensary_crawl_jobs.locked_by IS 'Hostname of worker that locked job';
-- ============================================================
-- 2. CREATE INDEXES FOR JOB QUEUE QUERIES
-- ============================================================
CREATE INDEX IF NOT EXISTS idx_crawl_jobs_status_priority
ON dispensary_crawl_jobs(status, priority DESC)
WHERE status = 'pending';
CREATE INDEX IF NOT EXISTS idx_crawl_jobs_worker_id
ON dispensary_crawl_jobs(worker_id)
WHERE worker_id IS NOT NULL;
CREATE INDEX IF NOT EXISTS idx_crawl_jobs_locked_at
ON dispensary_crawl_jobs(locked_at)
WHERE locked_at IS NOT NULL;
-- ============================================================
-- 3. CREATE QUEUE STATS VIEW
-- ============================================================
CREATE OR REPLACE VIEW v_queue_stats AS
SELECT
COUNT(*) FILTER (WHERE status = 'pending') AS pending_jobs,
COUNT(*) FILTER (WHERE status = 'running') AS running_jobs,
COUNT(*) FILTER (WHERE status = 'completed' AND completed_at > NOW() - INTERVAL '1 hour') AS completed_1h,
COUNT(*) FILTER (WHERE status = 'failed' AND completed_at > NOW() - INTERVAL '1 hour') AS failed_1h,
COUNT(DISTINCT worker_id) FILTER (WHERE status = 'running') AS active_workers,
ROUND((AVG(EXTRACT(EPOCH FROM (completed_at - started_at)))
FILTER (WHERE status = 'completed' AND completed_at > NOW() - INTERVAL '1 hour'))::numeric, 2) AS avg_duration_seconds
FROM dispensary_crawl_jobs;
COMMENT ON VIEW v_queue_stats IS 'Real-time queue statistics for monitoring dashboard';
-- ============================================================
-- 4. RECORD MIGRATION
-- ============================================================
INSERT INTO schema_migrations (version, name, applied_at)
VALUES (59, '059_job_queue_columns', NOW())
ON CONFLICT (version) DO NOTHING;