Update admin panel to use unified dispensaries table
- Add migration 026 to update dispensary_crawl_status view with new fields - Update dashboard API to use dispensaries table (not stores) - Show current inventory counts (products seen in last 7 days) - Update ScraperSchedule UI to show provider_type correctly 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
53
backend/migrations/026_update_crawl_status_view.sql
Normal file
53
backend/migrations/026_update_crawl_status_view.sql
Normal file
@@ -0,0 +1,53 @@
|
||||
-- Migration 026: Update dispensary_crawl_status view
|
||||
-- Use provider_type and scrape_enabled from dispensaries table (migration 025)
|
||||
|
||||
DROP VIEW IF EXISTS dispensary_crawl_status;
|
||||
|
||||
CREATE OR REPLACE VIEW dispensary_crawl_status AS
|
||||
SELECT
|
||||
d.id AS dispensary_id,
|
||||
COALESCE(d.dba_name, d.name) AS dispensary_name,
|
||||
d.city,
|
||||
d.state,
|
||||
d.slug,
|
||||
d.website,
|
||||
d.menu_url,
|
||||
-- Use provider_type from 025 if product_provider is null
|
||||
COALESCE(d.product_provider, d.provider_type) AS product_provider,
|
||||
d.provider_type,
|
||||
d.product_confidence,
|
||||
d.product_crawler_mode,
|
||||
d.last_product_scan_at,
|
||||
-- Use scrape_enabled from 025 if no schedule exists
|
||||
COALESCE(dcs.is_active, d.scrape_enabled, FALSE) AS schedule_active,
|
||||
COALESCE(dcs.interval_minutes, 240) AS interval_minutes,
|
||||
COALESCE(dcs.priority, 0) AS priority,
|
||||
-- Use crawl timestamps from 025 if no schedule exists
|
||||
COALESCE(dcs.last_run_at, d.last_crawl_at) AS last_run_at,
|
||||
COALESCE(dcs.next_run_at, d.next_crawl_at) AS next_run_at,
|
||||
COALESCE(dcs.last_status, d.crawl_status) AS last_status,
|
||||
dcs.last_summary,
|
||||
COALESCE(dcs.last_error, d.crawl_error) AS last_error,
|
||||
COALESCE(dcs.consecutive_failures, d.consecutive_failures, 0) AS consecutive_failures,
|
||||
COALESCE(dcs.total_runs, d.total_crawls, 0) AS total_runs,
|
||||
COALESCE(dcs.successful_runs, d.successful_crawls, 0) AS successful_runs,
|
||||
dcj.id AS latest_job_id,
|
||||
dcj.job_type AS latest_job_type,
|
||||
dcj.status AS latest_job_status,
|
||||
dcj.started_at AS latest_job_started,
|
||||
dcj.products_found AS latest_products_found
|
||||
FROM dispensaries d
|
||||
LEFT JOIN dispensary_crawl_schedule dcs ON dcs.dispensary_id = d.id
|
||||
LEFT JOIN LATERAL (
|
||||
SELECT * FROM dispensary_crawl_jobs
|
||||
WHERE dispensary_id = d.id
|
||||
ORDER BY created_at DESC
|
||||
LIMIT 1
|
||||
) dcj ON true
|
||||
ORDER BY
|
||||
CASE WHEN d.scrape_enabled = TRUE THEN 0 ELSE 1 END,
|
||||
COALESCE(dcs.priority, 0) DESC,
|
||||
COALESCE(d.dba_name, d.name);
|
||||
|
||||
-- Grant permissions
|
||||
GRANT SELECT ON dispensary_crawl_status TO dutchie;
|
||||
@@ -8,58 +8,63 @@ router.use(authMiddleware);
|
||||
// Get dashboard stats
|
||||
router.get('/stats', async (req, res) => {
|
||||
try {
|
||||
// Store stats
|
||||
const storesResult = await pool.query(`
|
||||
SELECT
|
||||
// Dispensary stats (using unified dispensaries table)
|
||||
const dispensariesResult = await pool.query(`
|
||||
SELECT
|
||||
COUNT(*) as total,
|
||||
COUNT(*) FILTER (WHERE active = true) as active,
|
||||
MIN(last_scraped_at) as oldest_scrape,
|
||||
MAX(last_scraped_at) as latest_scrape
|
||||
FROM stores
|
||||
COUNT(*) FILTER (WHERE scrape_enabled = true) as active,
|
||||
COUNT(*) FILTER (WHERE menu_url IS NOT NULL) as with_menu_url,
|
||||
COUNT(*) FILTER (WHERE provider_type IS NOT NULL AND provider_type != 'unknown') as detected,
|
||||
MIN(last_crawl_at) as oldest_crawl,
|
||||
MAX(last_crawl_at) as latest_crawl
|
||||
FROM dispensaries
|
||||
`);
|
||||
|
||||
// Product stats
|
||||
|
||||
// Current product stats (active inventory only)
|
||||
const productsResult = await pool.query(`
|
||||
SELECT
|
||||
SELECT
|
||||
COUNT(*) as total,
|
||||
COUNT(*) FILTER (WHERE in_stock = true) as in_stock,
|
||||
COUNT(*) FILTER (WHERE local_image_path IS NOT NULL) as with_images
|
||||
COUNT(*) FILTER (WHERE local_image_path IS NOT NULL) as with_images,
|
||||
COUNT(DISTINCT brand) as unique_brands,
|
||||
COUNT(DISTINCT dispensary_id) as stores_with_products
|
||||
FROM products
|
||||
WHERE last_seen_at >= NOW() - INTERVAL '7 days'
|
||||
`);
|
||||
|
||||
|
||||
// Campaign stats
|
||||
const campaignsResult = await pool.query(`
|
||||
SELECT
|
||||
SELECT
|
||||
COUNT(*) as total,
|
||||
COUNT(*) FILTER (WHERE active = true) as active
|
||||
FROM campaigns
|
||||
`);
|
||||
|
||||
|
||||
// Recent clicks (last 24 hours)
|
||||
const clicksResult = await pool.query(`
|
||||
SELECT COUNT(*) as clicks_24h
|
||||
FROM clicks
|
||||
WHERE clicked_at >= NOW() - INTERVAL '24 hours'
|
||||
`);
|
||||
|
||||
|
||||
// Recent products added (last 24 hours)
|
||||
const recentProductsResult = await pool.query(`
|
||||
SELECT COUNT(*) as new_products_24h
|
||||
FROM products
|
||||
WHERE first_seen_at >= NOW() - INTERVAL '24 hours'
|
||||
`);
|
||||
|
||||
|
||||
// Proxy stats
|
||||
const proxiesResult = await pool.query(`
|
||||
SELECT
|
||||
SELECT
|
||||
COUNT(*) as total,
|
||||
COUNT(*) FILTER (WHERE active = true) as active,
|
||||
COUNT(*) FILTER (WHERE is_anonymous = true) as anonymous
|
||||
FROM proxies
|
||||
`);
|
||||
|
||||
|
||||
res.json({
|
||||
stores: storesResult.rows[0],
|
||||
stores: dispensariesResult.rows[0], // Keep 'stores' key for frontend compat
|
||||
products: productsResult.rows[0],
|
||||
campaigns: campaignsResult.rows[0],
|
||||
clicks: clicksResult.rows[0],
|
||||
@@ -76,28 +81,28 @@ router.get('/stats', async (req, res) => {
|
||||
router.get('/activity', async (req, res) => {
|
||||
try {
|
||||
const { limit = 20 } = req.query;
|
||||
|
||||
// Recent scrapes
|
||||
|
||||
// Recent crawls from dispensaries
|
||||
const scrapesResult = await pool.query(`
|
||||
SELECT s.name, s.last_scraped_at,
|
||||
COUNT(p.id) as product_count
|
||||
FROM stores s
|
||||
LEFT JOIN products p ON s.id = p.store_id AND p.last_seen_at = s.last_scraped_at
|
||||
WHERE s.last_scraped_at IS NOT NULL
|
||||
GROUP BY s.id, s.name, s.last_scraped_at
|
||||
ORDER BY s.last_scraped_at DESC
|
||||
SELECT
|
||||
COALESCE(d.dba_name, d.name) as name,
|
||||
d.last_crawl_at as last_scraped_at,
|
||||
(SELECT COUNT(*) FROM products p WHERE p.dispensary_id = d.id AND p.last_seen_at >= NOW() - INTERVAL '7 days') as product_count
|
||||
FROM dispensaries d
|
||||
WHERE d.last_crawl_at IS NOT NULL
|
||||
ORDER BY d.last_crawl_at DESC
|
||||
LIMIT $1
|
||||
`, [limit]);
|
||||
|
||||
|
||||
// Recent products
|
||||
const productsResult = await pool.query(`
|
||||
SELECT p.name, p.price, s.name as store_name, p.first_seen_at
|
||||
SELECT p.name, p.price, COALESCE(d.dba_name, d.name) as store_name, p.first_seen_at
|
||||
FROM products p
|
||||
JOIN stores s ON p.store_id = s.id
|
||||
JOIN dispensaries d ON p.dispensary_id = d.id
|
||||
ORDER BY p.first_seen_at DESC
|
||||
LIMIT $1
|
||||
`, [limit]);
|
||||
|
||||
|
||||
res.json({
|
||||
recent_scrapes: scrapesResult.rows,
|
||||
recent_products: productsResult.rows
|
||||
|
||||
Reference in New Issue
Block a user