/** * Markets API Routes * * Provider-agnostic store and product endpoints for the CannaiQ admin dashboard. * Queries the dispensaries and dutchie_products tables directly. */ import { Router, Request, Response } from 'express'; import { authMiddleware } from '../auth/middleware'; import { pool } from '../db/pool'; const router = Router(); router.use(authMiddleware); /** * GET /api/markets/dashboard * Dashboard summary with counts for dispensaries, products, brands, etc. */ router.get('/dashboard', async (req: Request, res: Response) => { try { // Get dispensary count const { rows: dispRows } = await pool.query( `SELECT COUNT(*) as count FROM dispensaries` ); // Get product count from store_products (canonical) or fallback to dutchie_products const { rows: productRows } = await pool.query(` SELECT COUNT(*) as count FROM store_products `); // Get brand count const { rows: brandRows } = await pool.query(` SELECT COUNT(DISTINCT brand_name_raw) as count FROM store_products WHERE brand_name_raw IS NOT NULL `); // Get category count const { rows: categoryRows } = await pool.query(` SELECT COUNT(DISTINCT category_raw) as count FROM store_products WHERE category_raw IS NOT NULL `); // Get snapshot count in last 24 hours const { rows: snapshotRows } = await pool.query(` SELECT COUNT(*) as count FROM store_product_snapshots WHERE captured_at >= NOW() - INTERVAL '24 hours' `); // Get last crawl time const { rows: lastCrawlRows } = await pool.query(` SELECT MAX(completed_at) as last_crawl FROM crawl_orchestration_traces WHERE success = true `); // Get failed job count (jobs in last 24h that failed) const { rows: failedRows } = await pool.query(` SELECT COUNT(*) as count FROM crawl_orchestration_traces WHERE success = false AND started_at >= NOW() - INTERVAL '24 hours' `); res.json({ dispensaryCount: parseInt(dispRows[0]?.count || '0', 10), productCount: parseInt(productRows[0]?.count || '0', 10), brandCount: parseInt(brandRows[0]?.count || '0', 10), categoryCount: parseInt(categoryRows[0]?.count || '0', 10), snapshotCount24h: parseInt(snapshotRows[0]?.count || '0', 10), lastCrawlTime: lastCrawlRows[0]?.last_crawl || null, failedJobCount: parseInt(failedRows[0]?.count || '0', 10), }); } catch (error: any) { console.error('[Markets] Error fetching dashboard:', error.message); res.status(500).json({ error: error.message }); } }); /** * GET /api/markets/stores * List all stores from the dispensaries table */ router.get('/stores', async (req: Request, res: Response) => { try { const { city, hasPlatformId, limit = '100', offset = '0' } = req.query; let whereClause = 'WHERE 1=1'; const params: any[] = []; let paramIndex = 1; if (city) { whereClause += ` AND d.city ILIKE $${paramIndex}`; params.push(`%${city}%`); paramIndex++; } if (hasPlatformId === 'true') { whereClause += ` AND d.platform_dispensary_id IS NOT NULL`; } else if (hasPlatformId === 'false') { whereClause += ` AND d.platform_dispensary_id IS NULL`; } params.push(parseInt(limit as string, 10), parseInt(offset as string, 10)); const { rows } = await pool.query(` SELECT d.id, d.name, d.dba_name, d.city, d.state, d.address1 as address, d.zipcode as zip, d.phone, d.website, d.menu_url, d.menu_type, d.platform_dispensary_id, d.crawl_enabled, d.dutchie_verified, d.last_crawl_at, d.product_count, d.created_at, d.updated_at FROM dispensaries d ${whereClause} ORDER BY d.name LIMIT $${paramIndex} OFFSET $${paramIndex + 1} `, params); // Get total count const { rows: countRows } = await pool.query( `SELECT COUNT(*) as total FROM dispensaries d ${whereClause}`, params.slice(0, -2) ); res.json({ stores: rows, total: parseInt(countRows[0]?.total || '0', 10), }); } catch (error: any) { console.error('[Markets] Error fetching stores:', error.message); res.status(500).json({ error: error.message }); } }); /** * GET /api/markets/stores/:id * Get a single store by ID */ router.get('/stores/:id', async (req: Request, res: Response) => { try { const { id } = req.params; const { rows } = await pool.query(` SELECT d.id, d.name, d.dba_name, d.city, d.state, d.address1 as address, d.zipcode as zip, d.phone, d.website, d.menu_url, d.menu_type, d.platform_dispensary_id, d.crawl_enabled, d.dutchie_verified, d.last_crawl_at, d.product_count, d.created_at, d.updated_at FROM dispensaries d WHERE d.id = $1 `, [parseInt(id, 10)]); if (rows.length === 0) { return res.status(404).json({ error: 'Store not found' }); } res.json(rows[0]); } catch (error: any) { console.error('[Markets] Error fetching store:', error.message); res.status(500).json({ error: error.message }); } }); /** * GET /api/markets/stores/:id/summary * Get store summary with aggregated metrics, brands, and categories */ router.get('/stores/:id/summary', async (req: Request, res: Response) => { try { const { id } = req.params; const dispensaryId = parseInt(id, 10); // Get dispensary info const { rows: dispRows } = await pool.query(` SELECT d.id, d.name, d.dba_name, d.c_name as company_name, d.city, d.state, d.address1 as address, d.zipcode as zip, d.phone, d.website, d.menu_url, d.menu_type, d.platform_dispensary_id, d.crawl_enabled, d.last_crawl_at FROM dispensaries d WHERE d.id = $1 `, [dispensaryId]); if (dispRows.length === 0) { return res.status(404).json({ error: 'Store not found' }); } const dispensary = dispRows[0]; // Get product counts using canonical store_products table const { rows: countRows } = await pool.query(` SELECT COUNT(*) as total, COUNT(*) FILTER (WHERE stock_status = 'in_stock') as in_stock, COUNT(*) FILTER (WHERE stock_status = 'out_of_stock') as out_of_stock, COUNT(*) FILTER (WHERE stock_status NOT IN ('in_stock', 'out_of_stock') OR stock_status IS NULL) as unknown, COUNT(*) FILTER (WHERE stock_status = 'missing_from_feed') as missing_from_feed FROM store_products WHERE dispensary_id = $1 `, [dispensaryId]); const counts = countRows[0] || {}; // Get brands using canonical table const { rows: brandRows } = await pool.query(` SELECT brand_name_raw as brand_name, COUNT(*) as product_count FROM store_products WHERE dispensary_id = $1 AND brand_name_raw IS NOT NULL GROUP BY brand_name_raw ORDER BY product_count DESC, brand_name_raw `, [dispensaryId]); // Get categories using canonical table const { rows: categoryRows } = await pool.query(` SELECT category_raw as type, subcategory_raw as subcategory, COUNT(*) as product_count FROM store_products WHERE dispensary_id = $1 GROUP BY category_raw, subcategory_raw ORDER BY product_count DESC `, [dispensaryId]); // Get last crawl info from job_run_logs or crawl_orchestration_traces const { rows: crawlRows } = await pool.query(` SELECT completed_at, CASE WHEN success THEN 'completed' ELSE 'failed' END as status, error_message FROM crawl_orchestration_traces WHERE dispensary_id = $1 ORDER BY completed_at DESC LIMIT 1 `, [dispensaryId]); const lastCrawl = crawlRows.length > 0 ? crawlRows[0] : null; res.json({ dispensary, totalProducts: parseInt(counts.total || '0', 10), inStockCount: parseInt(counts.in_stock || '0', 10), outOfStockCount: parseInt(counts.out_of_stock || '0', 10), unknownStockCount: parseInt(counts.unknown || '0', 10), missingFromFeedCount: parseInt(counts.missing_from_feed || '0', 10), brands: brandRows, brandCount: brandRows.length, categories: categoryRows, categoryCount: categoryRows.length, lastCrawl, }); } catch (error: any) { console.error('[Markets] Error fetching store summary:', error.message); res.status(500).json({ error: error.message }); } }); /** * GET /api/markets/stores/:id/crawl-history * Get crawl history for a specific store */ router.get('/stores/:id/crawl-history', async (req: Request, res: Response) => { try { const { id } = req.params; const { limit = '50' } = req.query; const dispensaryId = parseInt(id, 10); const limitNum = Math.min(parseInt(limit as string, 10), 100); // Get crawl history from crawl_orchestration_traces const { rows: historyRows } = await pool.query(` SELECT id, run_id, profile_key, crawler_module, state_at_start, state_at_end, total_steps, duration_ms, success, error_message, products_found, started_at, completed_at FROM crawl_orchestration_traces WHERE dispensary_id = $1 ORDER BY started_at DESC LIMIT $2 `, [dispensaryId, limitNum]); // Get next scheduled crawl if available const { rows: scheduleRows } = await pool.query(` SELECT js.id as schedule_id, js.job_name, js.enabled, js.base_interval_minutes, js.jitter_minutes, js.next_run_at, js.last_run_at, js.last_status FROM job_schedules js WHERE js.enabled = true AND js.job_config->>'dispensaryId' = $1::text ORDER BY js.next_run_at LIMIT 1 `, [dispensaryId.toString()]); // Get dispensary info for slug const { rows: dispRows } = await pool.query(` SELECT id, name, dba_name, slug, state, city, menu_type, platform_dispensary_id, last_menu_scrape FROM dispensaries WHERE id = $1 `, [dispensaryId]); res.json({ dispensary: dispRows[0] || null, history: historyRows.map(row => ({ id: row.id, runId: row.run_id, profileKey: row.profile_key, crawlerModule: row.crawler_module, stateAtStart: row.state_at_start, stateAtEnd: row.state_at_end, totalSteps: row.total_steps, durationMs: row.duration_ms, success: row.success, errorMessage: row.error_message, productsFound: row.products_found, startedAt: row.started_at?.toISOString() || null, completedAt: row.completed_at?.toISOString() || null, })), nextSchedule: scheduleRows[0] ? { scheduleId: scheduleRows[0].schedule_id, jobName: scheduleRows[0].job_name, enabled: scheduleRows[0].enabled, baseIntervalMinutes: scheduleRows[0].base_interval_minutes, jitterMinutes: scheduleRows[0].jitter_minutes, nextRunAt: scheduleRows[0].next_run_at?.toISOString() || null, lastRunAt: scheduleRows[0].last_run_at?.toISOString() || null, lastStatus: scheduleRows[0].last_status, } : null, }); } catch (error: any) { console.error('[Markets] Error fetching crawl history:', error.message); res.status(500).json({ error: error.message }); } }); /** * GET /api/markets/stores/:id/products * Get products for a store with filtering and pagination */ router.get('/stores/:id/products', async (req: Request, res: Response) => { try { const { id } = req.params; const { stockStatus, type, subcategory, brandName, search, limit = '25', offset = '0' } = req.query; const dispensaryId = parseInt(id, 10); let whereClause = 'WHERE sp.dispensary_id = $1'; const params: any[] = [dispensaryId]; let paramIndex = 2; if (stockStatus) { whereClause += ` AND sp.stock_status = $${paramIndex}`; params.push(stockStatus); paramIndex++; } if (type) { whereClause += ` AND sp.category_raw = $${paramIndex}`; params.push(type); paramIndex++; } if (subcategory) { whereClause += ` AND sp.subcategory_raw = $${paramIndex}`; params.push(subcategory); paramIndex++; } if (brandName) { whereClause += ` AND sp.brand_name_raw ILIKE $${paramIndex}`; params.push(`%${brandName}%`); paramIndex++; } if (search) { whereClause += ` AND (sp.name_raw ILIKE $${paramIndex} OR sp.brand_name_raw ILIKE $${paramIndex})`; params.push(`%${search}%`); paramIndex++; } const limitNum = Math.min(parseInt(limit as string, 10), 100); const offsetNum = parseInt(offset as string, 10); params.push(limitNum, offsetNum); // Get products with latest snapshot data using canonical tables const { rows } = await pool.query(` SELECT sp.id, sp.external_product_id as external_id, sp.name_raw as name, sp.brand_name_raw as brand, sp.category_raw as type, sp.subcategory_raw as subcategory, sp.strain_type, sp.stock_status, sp.stock_status = 'in_stock' as in_stock, sp.stock_status != 'missing_from_feed' as is_present_in_feed, sp.stock_status = 'missing_from_feed' as missing_from_feed, sp.thc_percent as thc_percentage, sp.cbd_percent as cbd_percentage, sp.primary_image_url as image_url, sp.description, sp.total_quantity_available as total_quantity, sp.first_seen_at, sp.last_seen_at, sp.updated_at, ( SELECT jsonb_build_object( 'regular_price', COALESCE(sps.price_rec, 0)::numeric, 'sale_price', CASE WHEN sps.price_rec_special > 0 THEN sps.price_rec_special::numeric ELSE NULL END, 'med_price', COALESCE(sps.price_med, 0)::numeric, 'med_sale_price', CASE WHEN sps.price_med_special > 0 THEN sps.price_med_special::numeric ELSE NULL END, 'snapshot_at', sps.captured_at ) FROM store_product_snapshots sps WHERE sps.store_product_id = sp.id ORDER BY sps.captured_at DESC LIMIT 1 ) as pricing FROM store_products sp ${whereClause} ORDER BY sp.name_raw LIMIT $${paramIndex} OFFSET $${paramIndex + 1} `, params); // Flatten pricing into the product object const products = rows.map((row: any) => { const pricing = row.pricing || {}; return { ...row, regular_price: pricing.regular_price || null, sale_price: pricing.sale_price || null, med_price: pricing.med_price || null, med_sale_price: pricing.med_sale_price || null, snapshot_at: pricing.snapshot_at || null, pricing: undefined, // Remove the nested object }; }); // Get total count const { rows: countRows } = await pool.query( `SELECT COUNT(*) as total FROM store_products sp ${whereClause}`, params.slice(0, -2) ); res.json({ products, total: parseInt(countRows[0]?.total || '0', 10), limit: limitNum, offset: offsetNum, }); } catch (error: any) { console.error('[Markets] Error fetching store products:', error.message); res.status(500).json({ error: error.message }); } }); /** * GET /api/markets/stores/:id/brands * Get brands for a store */ router.get('/stores/:id/brands', async (req: Request, res: Response) => { try { const { id } = req.params; const dispensaryId = parseInt(id, 10); const { rows } = await pool.query(` SELECT brand_name_raw as brand, COUNT(*) as product_count FROM store_products WHERE dispensary_id = $1 AND brand_name_raw IS NOT NULL GROUP BY brand_name_raw ORDER BY product_count DESC, brand_name_raw `, [dispensaryId]); res.json({ brands: rows }); } catch (error: any) { console.error('[Markets] Error fetching store brands:', error.message); res.status(500).json({ error: error.message }); } }); /** * GET /api/markets/stores/:id/categories * Get categories for a store */ router.get('/stores/:id/categories', async (req: Request, res: Response) => { try { const { id } = req.params; const dispensaryId = parseInt(id, 10); const { rows } = await pool.query(` SELECT category_raw as type, subcategory_raw as subcategory, COUNT(*) as product_count FROM store_products WHERE dispensary_id = $1 GROUP BY category_raw, subcategory_raw ORDER BY product_count DESC `, [dispensaryId]); res.json({ categories: rows }); } catch (error: any) { console.error('[Markets] Error fetching store categories:', error.message); res.status(500).json({ error: error.message }); } }); /** * POST /api/markets/stores/:id/crawl * Trigger a crawl for a store (alias for existing crawl endpoint) */ router.post('/stores/:id/crawl', async (req: Request, res: Response) => { try { const { id } = req.params; const dispensaryId = parseInt(id, 10); // Verify store exists and has platform_dispensary_id const { rows } = await pool.query(` SELECT id, name, platform_dispensary_id, menu_type FROM dispensaries WHERE id = $1 `, [dispensaryId]); if (rows.length === 0) { return res.status(404).json({ error: 'Store not found' }); } const store = rows[0]; if (!store.platform_dispensary_id) { return res.status(400).json({ error: 'Store does not have a platform ID resolved. Cannot crawl.', store: { id: store.id, name: store.name, menu_type: store.menu_type } }); } // Insert a job into the crawl queue await pool.query(` INSERT INTO crawl_jobs (dispensary_id, job_type, status, created_at) VALUES ($1, 'dutchie_product_crawl', 'pending', NOW()) `, [dispensaryId]); res.json({ success: true, message: `Crawl queued for ${store.name}`, store: { id: store.id, name: store.name } }); } catch (error: any) { console.error('[Markets] Error triggering crawl:', error.message); res.status(500).json({ error: error.message }); } }); /** * GET /api/markets/brands * List all brands with product counts and store presence */ router.get('/brands', async (req: Request, res: Response) => { try { const { search, limit = '100', offset = '0', sortBy = 'products' } = req.query; const limitNum = Math.min(parseInt(limit as string, 10), 500); const offsetNum = parseInt(offset as string, 10); let whereClause = 'WHERE brand_name_raw IS NOT NULL AND brand_name_raw != \'\''; const params: any[] = []; let paramIndex = 1; if (search) { whereClause += ` AND brand_name_raw ILIKE $${paramIndex}`; params.push(`%${search}%`); paramIndex++; } // Determine sort column let orderBy = 'product_count DESC'; if (sortBy === 'stores') { orderBy = 'store_count DESC'; } else if (sortBy === 'name') { orderBy = 'brand_name ASC'; } params.push(limitNum, offsetNum); const { rows } = await pool.query(` SELECT brand_name_raw as brand_name, COUNT(*) as product_count, COUNT(DISTINCT dispensary_id) as store_count, AVG(price_rec) FILTER (WHERE price_rec > 0) as avg_price, array_agg(DISTINCT category_raw) FILTER (WHERE category_raw IS NOT NULL) as categories, MIN(first_seen_at) as first_seen_at, MAX(last_seen_at) as last_seen_at FROM store_products ${whereClause} GROUP BY brand_name_raw ORDER BY ${orderBy} LIMIT $${paramIndex} OFFSET $${paramIndex + 1} `, params); // Get total count const { rows: countRows } = await pool.query(` SELECT COUNT(DISTINCT brand_name_raw) as total FROM store_products ${whereClause} `, params.slice(0, -2)); // Calculate summary stats const { rows: summaryRows } = await pool.query(` SELECT COUNT(DISTINCT brand_name_raw) as total_brands, AVG(product_count) as avg_products_per_brand FROM ( SELECT brand_name_raw, COUNT(*) as product_count FROM store_products WHERE brand_name_raw IS NOT NULL AND brand_name_raw != '' GROUP BY brand_name_raw ) brand_counts `); res.json({ brands: rows.map((r: any, idx: number) => ({ id: idx + 1 + offsetNum, name: r.brand_name, normalized_name: null, product_count: parseInt(r.product_count, 10), store_count: parseInt(r.store_count, 10), avg_price: r.avg_price ? parseFloat(r.avg_price) : null, categories: r.categories || [], is_portfolio: false, first_seen_at: r.first_seen_at, last_seen_at: r.last_seen_at, })), total: parseInt(countRows[0]?.total || '0', 10), summary: { total_brands: parseInt(summaryRows[0]?.total_brands || '0', 10), portfolio_brands: 0, avg_products_per_brand: Math.round(parseFloat(summaryRows[0]?.avg_products_per_brand || '0')), top_categories: [], }, limit: limitNum, offset: offsetNum, }); } catch (error: any) { console.error('[Markets] Error fetching brands:', error.message); res.status(500).json({ error: error.message }); } }); /** * GET /api/markets/categories * List all categories with product counts */ router.get('/categories', async (req: Request, res: Response) => { try { const { search, limit = '100' } = req.query; const limitNum = Math.min(parseInt(limit as string, 10), 500); let whereClause = 'WHERE category_raw IS NOT NULL AND category_raw != \'\''; const params: any[] = []; let paramIndex = 1; if (search) { whereClause += ` AND category_raw ILIKE $${paramIndex}`; params.push(`%${search}%`); paramIndex++; } params.push(limitNum); const { rows } = await pool.query(` SELECT category_raw as name, COUNT(*) as product_count, COUNT(DISTINCT dispensary_id) as store_count, AVG(price_rec) FILTER (WHERE price_rec > 0) as avg_price FROM store_products ${whereClause} GROUP BY category_raw ORDER BY product_count DESC LIMIT $${paramIndex} `, params); res.json({ categories: rows.map((r: any, idx: number) => ({ id: idx + 1, name: r.name, product_count: parseInt(r.product_count, 10), store_count: parseInt(r.store_count, 10), avg_price: r.avg_price ? parseFloat(r.avg_price) : null, })), total: rows.length, }); } catch (error: any) { console.error('[Markets] Error fetching categories:', error.message); res.status(500).json({ error: error.message }); } }); export default router;