From 9de0d709b29e2f75f4b9d53aca0210316a303009 Mon Sep 17 00:00:00 2001 From: Kelly Date: Mon, 1 Dec 2025 00:13:41 -0700 Subject: [PATCH] Update admin panel to use unified dispensaries table MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add migration 026 to update dispensary_crawl_status view with new fields - Update dashboard API to use dispensaries table (not stores) - Show current inventory counts (products seen in last 7 days) - Update ScraperSchedule UI to show provider_type correctly 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- .../026_update_crawl_status_view.sql | 53 ++++++++++++++ backend/src/routes/dashboard.ts | 69 ++++++++++--------- frontend/src/pages/ScraperSchedule.tsx | 18 ++++- 3 files changed, 105 insertions(+), 35 deletions(-) create mode 100644 backend/migrations/026_update_crawl_status_view.sql diff --git a/backend/migrations/026_update_crawl_status_view.sql b/backend/migrations/026_update_crawl_status_view.sql new file mode 100644 index 00000000..ec90e697 --- /dev/null +++ b/backend/migrations/026_update_crawl_status_view.sql @@ -0,0 +1,53 @@ +-- Migration 026: Update dispensary_crawl_status view +-- Use provider_type and scrape_enabled from dispensaries table (migration 025) + +DROP VIEW IF EXISTS dispensary_crawl_status; + +CREATE OR REPLACE VIEW dispensary_crawl_status AS +SELECT + d.id AS dispensary_id, + COALESCE(d.dba_name, d.name) AS dispensary_name, + d.city, + d.state, + d.slug, + d.website, + d.menu_url, + -- Use provider_type from 025 if product_provider is null + COALESCE(d.product_provider, d.provider_type) AS product_provider, + d.provider_type, + d.product_confidence, + d.product_crawler_mode, + d.last_product_scan_at, + -- Use scrape_enabled from 025 if no schedule exists + COALESCE(dcs.is_active, d.scrape_enabled, FALSE) AS schedule_active, + COALESCE(dcs.interval_minutes, 240) AS interval_minutes, + COALESCE(dcs.priority, 0) AS priority, + -- Use crawl timestamps from 025 if no schedule exists + COALESCE(dcs.last_run_at, d.last_crawl_at) AS last_run_at, + COALESCE(dcs.next_run_at, d.next_crawl_at) AS next_run_at, + COALESCE(dcs.last_status, d.crawl_status) AS last_status, + dcs.last_summary, + COALESCE(dcs.last_error, d.crawl_error) AS last_error, + COALESCE(dcs.consecutive_failures, d.consecutive_failures, 0) AS consecutive_failures, + COALESCE(dcs.total_runs, d.total_crawls, 0) AS total_runs, + COALESCE(dcs.successful_runs, d.successful_crawls, 0) AS successful_runs, + dcj.id AS latest_job_id, + dcj.job_type AS latest_job_type, + dcj.status AS latest_job_status, + dcj.started_at AS latest_job_started, + dcj.products_found AS latest_products_found +FROM dispensaries d +LEFT JOIN dispensary_crawl_schedule dcs ON dcs.dispensary_id = d.id +LEFT JOIN LATERAL ( + SELECT * FROM dispensary_crawl_jobs + WHERE dispensary_id = d.id + ORDER BY created_at DESC + LIMIT 1 +) dcj ON true +ORDER BY + CASE WHEN d.scrape_enabled = TRUE THEN 0 ELSE 1 END, + COALESCE(dcs.priority, 0) DESC, + COALESCE(d.dba_name, d.name); + +-- Grant permissions +GRANT SELECT ON dispensary_crawl_status TO dutchie; diff --git a/backend/src/routes/dashboard.ts b/backend/src/routes/dashboard.ts index e3944919..2f04dbc5 100755 --- a/backend/src/routes/dashboard.ts +++ b/backend/src/routes/dashboard.ts @@ -8,58 +8,63 @@ router.use(authMiddleware); // Get dashboard stats router.get('/stats', async (req, res) => { try { - // Store stats - const storesResult = await pool.query(` - SELECT + // Dispensary stats (using unified dispensaries table) + const dispensariesResult = await pool.query(` + SELECT COUNT(*) as total, - COUNT(*) FILTER (WHERE active = true) as active, - MIN(last_scraped_at) as oldest_scrape, - MAX(last_scraped_at) as latest_scrape - FROM stores + COUNT(*) FILTER (WHERE scrape_enabled = true) as active, + COUNT(*) FILTER (WHERE menu_url IS NOT NULL) as with_menu_url, + COUNT(*) FILTER (WHERE provider_type IS NOT NULL AND provider_type != 'unknown') as detected, + MIN(last_crawl_at) as oldest_crawl, + MAX(last_crawl_at) as latest_crawl + FROM dispensaries `); - - // Product stats + + // Current product stats (active inventory only) const productsResult = await pool.query(` - SELECT + SELECT COUNT(*) as total, COUNT(*) FILTER (WHERE in_stock = true) as in_stock, - COUNT(*) FILTER (WHERE local_image_path IS NOT NULL) as with_images + COUNT(*) FILTER (WHERE local_image_path IS NOT NULL) as with_images, + COUNT(DISTINCT brand) as unique_brands, + COUNT(DISTINCT dispensary_id) as stores_with_products FROM products + WHERE last_seen_at >= NOW() - INTERVAL '7 days' `); - + // Campaign stats const campaignsResult = await pool.query(` - SELECT + SELECT COUNT(*) as total, COUNT(*) FILTER (WHERE active = true) as active FROM campaigns `); - + // Recent clicks (last 24 hours) const clicksResult = await pool.query(` SELECT COUNT(*) as clicks_24h FROM clicks WHERE clicked_at >= NOW() - INTERVAL '24 hours' `); - + // Recent products added (last 24 hours) const recentProductsResult = await pool.query(` SELECT COUNT(*) as new_products_24h FROM products WHERE first_seen_at >= NOW() - INTERVAL '24 hours' `); - + // Proxy stats const proxiesResult = await pool.query(` - SELECT + SELECT COUNT(*) as total, COUNT(*) FILTER (WHERE active = true) as active, COUNT(*) FILTER (WHERE is_anonymous = true) as anonymous FROM proxies `); - + res.json({ - stores: storesResult.rows[0], + stores: dispensariesResult.rows[0], // Keep 'stores' key for frontend compat products: productsResult.rows[0], campaigns: campaignsResult.rows[0], clicks: clicksResult.rows[0], @@ -76,28 +81,28 @@ router.get('/stats', async (req, res) => { router.get('/activity', async (req, res) => { try { const { limit = 20 } = req.query; - - // Recent scrapes + + // Recent crawls from dispensaries const scrapesResult = await pool.query(` - SELECT s.name, s.last_scraped_at, - COUNT(p.id) as product_count - FROM stores s - LEFT JOIN products p ON s.id = p.store_id AND p.last_seen_at = s.last_scraped_at - WHERE s.last_scraped_at IS NOT NULL - GROUP BY s.id, s.name, s.last_scraped_at - ORDER BY s.last_scraped_at DESC + SELECT + COALESCE(d.dba_name, d.name) as name, + d.last_crawl_at as last_scraped_at, + (SELECT COUNT(*) FROM products p WHERE p.dispensary_id = d.id AND p.last_seen_at >= NOW() - INTERVAL '7 days') as product_count + FROM dispensaries d + WHERE d.last_crawl_at IS NOT NULL + ORDER BY d.last_crawl_at DESC LIMIT $1 `, [limit]); - + // Recent products const productsResult = await pool.query(` - SELECT p.name, p.price, s.name as store_name, p.first_seen_at + SELECT p.name, p.price, COALESCE(d.dba_name, d.name) as store_name, p.first_seen_at FROM products p - JOIN stores s ON p.store_id = s.id + JOIN dispensaries d ON p.dispensary_id = d.id ORDER BY p.first_seen_at DESC LIMIT $1 `, [limit]); - + res.json({ recent_scrapes: scrapesResult.rows, recent_products: productsResult.rows diff --git a/frontend/src/pages/ScraperSchedule.tsx b/frontend/src/pages/ScraperSchedule.tsx index 71fe88b6..307ae832 100644 --- a/frontend/src/pages/ScraperSchedule.tsx +++ b/frontend/src/pages/ScraperSchedule.tsx @@ -22,6 +22,7 @@ interface DispensarySchedule { website: string | null; menu_url: string | null; product_provider: string | null; + provider_type: string | null; product_confidence: number | null; product_crawler_mode: string | null; last_product_scan_at: string | null; @@ -415,7 +416,7 @@ export function ScraperSchedule() { - {disp.product_provider ? ( + {(disp.product_provider || disp.provider_type) && disp.product_provider !== 'unknown' && disp.provider_type !== 'unknown' ? (
- {disp.product_provider} + {disp.product_provider || disp.provider_type} {disp.product_crawler_mode !== 'production' && (
sandbox
)}
+ ) : disp.menu_url ? ( + + Pending + ) : ( - Unknown + - )}