feat(cannaiq): Add Workers Dashboard and visibility tracking
Workers Dashboard: - New /workers route with two-pane layout - Workers table showing Alice, Henry, Bella, Oscar with role badges - Run history with visibility stats (lost/restored counts) - "Run Now" action to trigger workers immediately Migrations: - 057: Add visibility tracking columns (visibility_lost, visibility_lost_at, visibility_restored_at) - 058: Add ID resolution columns for Henry worker - 059: Add job queue columns (max_retries, retry_count, worker_id, locked_at, locked_by) Backend fixes: - Add httpStatus to CrawlResult interface for error classification - Fix pool.ts typing for event listener - Update completeJob to accept visibility stats in metadata Frontend fixes: - Fix NationalDashboard crash with safe formatMoney helper - Fix OrchestratorDashboard/Stores StoreInfo type mismatches - Add workerName/workerRole to getDutchieAZSchedules API type 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
64
backend/migrations/057_visibility_tracking_columns.sql
Normal file
64
backend/migrations/057_visibility_tracking_columns.sql
Normal file
@@ -0,0 +1,64 @@
|
||||
-- Migration 057: Add visibility tracking columns to dutchie_products
|
||||
--
|
||||
-- Supports Bella (Product Sync) worker visibility-loss tracking:
|
||||
-- - visibility_lost: TRUE when product disappears from GraphQL feed
|
||||
-- - visibility_lost_at: Timestamp when product first went missing
|
||||
-- - visibility_restored_at: Timestamp when product reappeared
|
||||
--
|
||||
-- These columns enable tracking of products that temporarily or permanently
|
||||
-- disappear from Dutchie GraphQL API responses.
|
||||
|
||||
-- ============================================================
|
||||
-- 1. ADD VISIBILITY TRACKING COLUMNS TO dutchie_products
|
||||
-- ============================================================
|
||||
|
||||
ALTER TABLE dutchie_products
|
||||
ADD COLUMN IF NOT EXISTS visibility_lost BOOLEAN DEFAULT FALSE,
|
||||
ADD COLUMN IF NOT EXISTS visibility_lost_at TIMESTAMPTZ,
|
||||
ADD COLUMN IF NOT EXISTS visibility_restored_at TIMESTAMPTZ;
|
||||
|
||||
COMMENT ON COLUMN dutchie_products.visibility_lost IS 'TRUE when product is missing from GraphQL feed';
|
||||
COMMENT ON COLUMN dutchie_products.visibility_lost_at IS 'Timestamp when product first went missing from feed';
|
||||
COMMENT ON COLUMN dutchie_products.visibility_restored_at IS 'Timestamp when product reappeared after being missing';
|
||||
|
||||
-- ============================================================
|
||||
-- 2. CREATE INDEX FOR VISIBILITY QUERIES
|
||||
-- ============================================================
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_dutchie_products_visibility_lost
|
||||
ON dutchie_products(visibility_lost)
|
||||
WHERE visibility_lost = TRUE;
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_dutchie_products_visibility_lost_at
|
||||
ON dutchie_products(visibility_lost_at)
|
||||
WHERE visibility_lost_at IS NOT NULL;
|
||||
|
||||
-- ============================================================
|
||||
-- 3. CREATE VIEW FOR VISIBILITY ANALYTICS
|
||||
-- ============================================================
|
||||
|
||||
CREATE OR REPLACE VIEW v_visibility_summary AS
|
||||
SELECT
|
||||
d.id AS dispensary_id,
|
||||
d.name AS dispensary_name,
|
||||
d.state,
|
||||
COUNT(dp.id) AS total_products,
|
||||
COUNT(dp.id) FILTER (WHERE dp.visibility_lost = TRUE) AS visibility_lost_count,
|
||||
COUNT(dp.id) FILTER (WHERE dp.visibility_lost = FALSE OR dp.visibility_lost IS NULL) AS visible_count,
|
||||
COUNT(dp.id) FILTER (WHERE dp.visibility_restored_at IS NOT NULL) AS restored_count,
|
||||
MAX(dp.visibility_lost_at) AS latest_loss_at,
|
||||
MAX(dp.visibility_restored_at) AS latest_restore_at
|
||||
FROM dispensaries d
|
||||
LEFT JOIN dutchie_products dp ON d.id = dp.dispensary_id
|
||||
WHERE d.menu_type = 'dutchie'
|
||||
GROUP BY d.id, d.name, d.state;
|
||||
|
||||
COMMENT ON VIEW v_visibility_summary IS 'Aggregated visibility metrics per dispensary for dashboard analytics';
|
||||
|
||||
-- ============================================================
|
||||
-- 4. RECORD MIGRATION
|
||||
-- ============================================================
|
||||
|
||||
INSERT INTO schema_migrations (version, name, applied_at)
|
||||
VALUES (57, '057_visibility_tracking_columns', NOW())
|
||||
ON CONFLICT (version) DO NOTHING;
|
||||
46
backend/migrations/058_add_id_resolution_columns.sql
Normal file
46
backend/migrations/058_add_id_resolution_columns.sql
Normal file
@@ -0,0 +1,46 @@
|
||||
-- Migration 058: Add ID resolution tracking columns to dispensaries
|
||||
--
|
||||
-- Supports Henry (Entry Point Finder) worker tracking:
|
||||
-- - id_resolution_attempts: Count of how many times we've tried to resolve platform ID
|
||||
-- - last_id_resolution_at: When we last tried (matches code expectation)
|
||||
-- - id_resolution_status: Current status (pending, resolved, failed)
|
||||
-- - id_resolution_error: Last error message from resolution attempt
|
||||
|
||||
-- ============================================================
|
||||
-- 1. ADD ID RESOLUTION COLUMNS TO dispensaries
|
||||
-- ============================================================
|
||||
|
||||
ALTER TABLE dispensaries
|
||||
ADD COLUMN IF NOT EXISTS id_resolution_attempts INTEGER DEFAULT 0,
|
||||
ADD COLUMN IF NOT EXISTS last_id_resolution_at TIMESTAMPTZ,
|
||||
ADD COLUMN IF NOT EXISTS id_resolution_status VARCHAR(20) DEFAULT 'pending',
|
||||
ADD COLUMN IF NOT EXISTS id_resolution_error TEXT;
|
||||
|
||||
COMMENT ON COLUMN dispensaries.id_resolution_attempts IS 'Number of attempts to resolve platform_dispensary_id';
|
||||
COMMENT ON COLUMN dispensaries.last_id_resolution_at IS 'Timestamp of last ID resolution attempt';
|
||||
COMMENT ON COLUMN dispensaries.id_resolution_status IS 'Status: pending, resolved, failed';
|
||||
COMMENT ON COLUMN dispensaries.id_resolution_error IS 'Last error message from ID resolution attempt';
|
||||
|
||||
-- Additional columns needed by worker/scheduler
|
||||
ALTER TABLE dispensaries
|
||||
ADD COLUMN IF NOT EXISTS failed_at TIMESTAMPTZ,
|
||||
ADD COLUMN IF NOT EXISTS failure_notes TEXT;
|
||||
|
||||
COMMENT ON COLUMN dispensaries.failed_at IS 'Timestamp when dispensary was marked as permanently failed';
|
||||
COMMENT ON COLUMN dispensaries.failure_notes IS 'Notes about why dispensary was marked as failed';
|
||||
|
||||
-- ============================================================
|
||||
-- 2. CREATE INDEX FOR RESOLUTION QUERIES
|
||||
-- ============================================================
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_dispensaries_id_resolution_status
|
||||
ON dispensaries(id_resolution_status)
|
||||
WHERE id_resolution_status = 'pending';
|
||||
|
||||
-- ============================================================
|
||||
-- 3. RECORD MIGRATION
|
||||
-- ============================================================
|
||||
|
||||
INSERT INTO schema_migrations (version, name, applied_at)
|
||||
VALUES (58, '058_add_id_resolution_columns', NOW())
|
||||
ON CONFLICT (version) DO NOTHING;
|
||||
67
backend/migrations/059_job_queue_columns.sql
Normal file
67
backend/migrations/059_job_queue_columns.sql
Normal file
@@ -0,0 +1,67 @@
|
||||
-- Migration 059: Add missing columns to dispensary_crawl_jobs
|
||||
--
|
||||
-- Required for worker job processing:
|
||||
-- - max_retries: Maximum retry attempts for a job
|
||||
-- - retry_count: Current retry count
|
||||
-- - worker_id: ID of worker processing the job
|
||||
-- - locked_at: When the job was locked by a worker
|
||||
-- - locked_by: Hostname of worker that locked the job
|
||||
|
||||
-- ============================================================
|
||||
-- 1. ADD JOB QUEUE COLUMNS
|
||||
-- ============================================================
|
||||
|
||||
ALTER TABLE dispensary_crawl_jobs
|
||||
ADD COLUMN IF NOT EXISTS max_retries INTEGER DEFAULT 3,
|
||||
ADD COLUMN IF NOT EXISTS retry_count INTEGER DEFAULT 0,
|
||||
ADD COLUMN IF NOT EXISTS worker_id VARCHAR(100),
|
||||
ADD COLUMN IF NOT EXISTS locked_at TIMESTAMPTZ,
|
||||
ADD COLUMN IF NOT EXISTS locked_by VARCHAR(100),
|
||||
ADD COLUMN IF NOT EXISTS updated_at TIMESTAMPTZ DEFAULT NOW();
|
||||
|
||||
COMMENT ON COLUMN dispensary_crawl_jobs.max_retries IS 'Maximum number of retry attempts';
|
||||
COMMENT ON COLUMN dispensary_crawl_jobs.retry_count IS 'Current retry count';
|
||||
COMMENT ON COLUMN dispensary_crawl_jobs.worker_id IS 'ID of worker processing this job';
|
||||
COMMENT ON COLUMN dispensary_crawl_jobs.locked_at IS 'When job was locked by worker';
|
||||
COMMENT ON COLUMN dispensary_crawl_jobs.locked_by IS 'Hostname of worker that locked job';
|
||||
|
||||
-- ============================================================
|
||||
-- 2. CREATE INDEXES FOR JOB QUEUE QUERIES
|
||||
-- ============================================================
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_crawl_jobs_status_priority
|
||||
ON dispensary_crawl_jobs(status, priority DESC)
|
||||
WHERE status = 'pending';
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_crawl_jobs_worker_id
|
||||
ON dispensary_crawl_jobs(worker_id)
|
||||
WHERE worker_id IS NOT NULL;
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_crawl_jobs_locked_at
|
||||
ON dispensary_crawl_jobs(locked_at)
|
||||
WHERE locked_at IS NOT NULL;
|
||||
|
||||
-- ============================================================
|
||||
-- 3. CREATE QUEUE STATS VIEW
|
||||
-- ============================================================
|
||||
|
||||
CREATE OR REPLACE VIEW v_queue_stats AS
|
||||
SELECT
|
||||
COUNT(*) FILTER (WHERE status = 'pending') AS pending_jobs,
|
||||
COUNT(*) FILTER (WHERE status = 'running') AS running_jobs,
|
||||
COUNT(*) FILTER (WHERE status = 'completed' AND completed_at > NOW() - INTERVAL '1 hour') AS completed_1h,
|
||||
COUNT(*) FILTER (WHERE status = 'failed' AND completed_at > NOW() - INTERVAL '1 hour') AS failed_1h,
|
||||
COUNT(DISTINCT worker_id) FILTER (WHERE status = 'running') AS active_workers,
|
||||
ROUND((AVG(EXTRACT(EPOCH FROM (completed_at - started_at)))
|
||||
FILTER (WHERE status = 'completed' AND completed_at > NOW() - INTERVAL '1 hour'))::numeric, 2) AS avg_duration_seconds
|
||||
FROM dispensary_crawl_jobs;
|
||||
|
||||
COMMENT ON VIEW v_queue_stats IS 'Real-time queue statistics for monitoring dashboard';
|
||||
|
||||
-- ============================================================
|
||||
-- 4. RECORD MIGRATION
|
||||
-- ============================================================
|
||||
|
||||
INSERT INTO schema_migrations (version, name, applied_at)
|
||||
VALUES (59, '059_job_queue_columns', NOW())
|
||||
ON CONFLICT (version) DO NOTHING;
|
||||
Reference in New Issue
Block a user