cannaiq/backend/src/routes/markets.ts

/**
 * Markets API Routes
 *
 * Provider-agnostic store and product endpoints for the CannaiQ admin dashboard.
 * Queries the dispensaries and dutchie_products tables directly.
 */
import { Router, Request, Response } from 'express';
import { authMiddleware } from '../auth/middleware';
import { pool } from '../db/pool';

const router = Router();
router.use(authMiddleware);

/**
 * GET /api/markets/dashboard
 * Dashboard summary with counts for dispensaries, products, brands, etc.
 */
router.get('/dashboard', async (req: Request, res: Response) => {
  try {
    // Get dispensary count
    const { rows: dispRows } = await pool.query(
      `SELECT COUNT(*) as count FROM dispensaries`
    );

    // Get product count from store_products (canonical) or fallback to dutchie_products
    const { rows: productRows } = await pool.query(`
      SELECT COUNT(*) as count FROM store_products
    `);

    // Get brand count
    const { rows: brandRows } = await pool.query(`
      SELECT COUNT(DISTINCT brand_name_raw) as count
      FROM store_products
      WHERE brand_name_raw IS NOT NULL
    `);

    // Get category count
    const { rows: categoryRows } = await pool.query(`
      SELECT COUNT(DISTINCT category_raw) as count
      FROM store_products
      WHERE category_raw IS NOT NULL
    `);

    // Get snapshot count in last 24 hours
    const { rows: snapshotRows } = await pool.query(`
      SELECT COUNT(*) as count
      FROM store_product_snapshots
      WHERE captured_at >= NOW() - INTERVAL '24 hours'
    `);

    // Get last crawl time
    const { rows: lastCrawlRows } = await pool.query(`
      SELECT MAX(completed_at) as last_crawl
      FROM crawl_orchestration_traces
      WHERE success = true
    `);

    // Get failed job count (jobs in last 24h that failed)
    const { rows: failedRows } = await pool.query(`
      SELECT COUNT(*) as count
      FROM crawl_orchestration_traces
      WHERE success = false
        AND started_at >= NOW() - INTERVAL '24 hours'
    `);

    res.json({
      dispensaryCount: parseInt(dispRows[0]?.count || '0', 10),
      productCount: parseInt(productRows[0]?.count || '0', 10),
      brandCount: parseInt(brandRows[0]?.count || '0', 10),
      categoryCount: parseInt(categoryRows[0]?.count || '0', 10),
      snapshotCount24h: parseInt(snapshotRows[0]?.count || '0', 10),
      lastCrawlTime: lastCrawlRows[0]?.last_crawl || null,
      failedJobCount: parseInt(failedRows[0]?.count || '0', 10),
    });
  } catch (error: any) {
    console.error('[Markets] Error fetching dashboard:', error.message);
    res.status(500).json({ error: error.message });
  }
});

/**
 * GET /api/markets/stores
 * List all stores from the dispensaries table
 */
router.get('/stores', async (req: Request, res: Response) => {
  try {
    const { city, hasPlatformId, limit = '100', offset = '0' } = req.query;

    let whereClause = 'WHERE 1=1';
    const params: any[] = [];
    let paramIndex = 1;

    if (city) {
      whereClause += ` AND d.city ILIKE $${paramIndex}`;
      params.push(`%${city}%`);
      paramIndex++;
    }

    if (hasPlatformId === 'true') {
      whereClause += ` AND d.platform_dispensary_id IS NOT NULL`;
    } else if (hasPlatformId === 'false') {
      whereClause += ` AND d.platform_dispensary_id IS NULL`;
    }

    params.push(parseInt(limit as string, 10), parseInt(offset as string, 10));

    const { rows } = await pool.query(`
      SELECT
        d.id,
        d.name,
        d.dba_name,
        d.city,
        d.state,
        d.address1 as address,
        d.zipcode as zip,
        d.phone,
        d.website,
        d.menu_url,
        d.menu_type,
        d.platform_dispensary_id,
        d.crawl_enabled,
        d.dutchie_verified,
        d.last_crawl_at,
        d.product_count,
        d.created_at,
        d.updated_at
      FROM dispensaries d
      ${whereClause}
      ORDER BY d.name
      LIMIT $${paramIndex} OFFSET $${paramIndex + 1}
    `, params);

    // Get total count
    const { rows: countRows } = await pool.query(
      `SELECT COUNT(*) as total FROM dispensaries d ${whereClause}`,
      params.slice(0, -2)
    );

    res.json({
      stores: rows,
      total: parseInt(countRows[0]?.total || '0', 10),
    });
  } catch (error: any) {
    console.error('[Markets] Error fetching stores:', error.message);
    res.status(500).json({ error: error.message });
  }
});

/**
 * GET /api/markets/stores/:id
 * Get a single store by ID
 */
router.get('/stores/:id', async (req: Request, res: Response) => {
  try {
    const { id } = req.params;

    const { rows } = await pool.query(`
      SELECT
        d.id,
        d.name,
        d.dba_name,
        d.city,
        d.state,
        d.address1 as address,
        d.zipcode as zip,
        d.phone,
        d.website,
        d.menu_url,
        d.menu_type,
        d.platform_dispensary_id,
        d.crawl_enabled,
        d.dutchie_verified,
        d.last_crawl_at,
        d.product_count,
        d.created_at,
        d.updated_at
      FROM dispensaries d
      WHERE d.id = $1
    `, [parseInt(id, 10)]);

    if (rows.length === 0) {
      return res.status(404).json({ error: 'Store not found' });
    }

    res.json(rows[0]);
  } catch (error: any) {
    console.error('[Markets] Error fetching store:', error.message);
    res.status(500).json({ error: error.message });
  }
});

/**
 * GET /api/markets/stores/:id/summary
 * Get store summary with aggregated metrics, brands, and categories
 */
router.get('/stores/:id/summary', async (req: Request, res: Response) => {
  try {
    const { id } = req.params;
    const dispensaryId = parseInt(id, 10);

    // Get dispensary info
    const { rows: dispRows } = await pool.query(`
      SELECT
        d.id,
        d.name,
        d.dba_name,
        d.c_name as company_name,
        d.city,
        d.state,
        d.address1 as address,
        d.zipcode as zip,
        d.phone,
        d.website,
        d.menu_url,
        d.menu_type,
        d.platform_dispensary_id,
        d.crawl_enabled,
        d.last_crawl_at
      FROM dispensaries d
      WHERE d.id = $1
    `, [dispensaryId]);

    if (dispRows.length === 0) {
      return res.status(404).json({ error: 'Store not found' });
    }

    const dispensary = dispRows[0];

    // Get product counts using canonical store_products table
    const { rows: countRows } = await pool.query(`
      SELECT
        COUNT(*) as total,
        COUNT(*) FILTER (WHERE stock_status = 'in_stock') as in_stock,
        COUNT(*) FILTER (WHERE stock_status = 'out_of_stock') as out_of_stock,
        COUNT(*) FILTER (WHERE stock_status NOT IN ('in_stock', 'out_of_stock') OR stock_status IS NULL) as unknown,
        COUNT(*) FILTER (WHERE stock_status = 'missing_from_feed') as missing_from_feed
      FROM store_products
      WHERE dispensary_id = $1
    `, [dispensaryId]);

    const counts = countRows[0] || {};

    // Get brands using canonical table
    const { rows: brandRows } = await pool.query(`
      SELECT brand_name_raw as brand_name, COUNT(*) as product_count
      FROM store_products
      WHERE dispensary_id = $1 AND brand_name_raw IS NOT NULL
      GROUP BY brand_name_raw
      ORDER BY product_count DESC, brand_name_raw
    `, [dispensaryId]);

    // Get categories using canonical table
    const { rows: categoryRows } = await pool.query(`
      SELECT category_raw as type, subcategory_raw as subcategory, COUNT(*) as product_count
      FROM store_products
      WHERE dispensary_id = $1
      GROUP BY category_raw, subcategory_raw
      ORDER BY product_count DESC
    `, [dispensaryId]);

    // Get last crawl info from job_run_logs or crawl_orchestration_traces
    const { rows: crawlRows } = await pool.query(`
      SELECT
        completed_at,
        CASE WHEN success THEN 'completed' ELSE 'failed' END as status,
        error_message
      FROM crawl_orchestration_traces
      WHERE dispensary_id = $1
      ORDER BY completed_at DESC
      LIMIT 1
    `, [dispensaryId]);

    const lastCrawl = crawlRows.length > 0 ? crawlRows[0] : null;

    res.json({
      dispensary,
      totalProducts: parseInt(counts.total || '0', 10),
      inStockCount: parseInt(counts.in_stock || '0', 10),
      outOfStockCount: parseInt(counts.out_of_stock || '0', 10),
      unknownStockCount: parseInt(counts.unknown || '0', 10),
      missingFromFeedCount: parseInt(counts.missing_from_feed || '0', 10),
      brands: brandRows,
      brandCount: brandRows.length,
      categories: categoryRows,
      categoryCount: categoryRows.length,
      lastCrawl,
    });
  } catch (error: any) {
    console.error('[Markets] Error fetching store summary:', error.message);
    res.status(500).json({ error: error.message });
  }
});

/**
 * GET /api/markets/stores/:id/crawl-history
 * Get crawl history for a specific store
 */
router.get('/stores/:id/crawl-history', async (req: Request, res: Response) => {
  try {
    const { id } = req.params;
    const { limit = '50' } = req.query;
    const dispensaryId = parseInt(id, 10);
    const limitNum = Math.min(parseInt(limit as string, 10), 100);

    // Get crawl history from crawl_orchestration_traces
    const { rows: historyRows } = await pool.query(`
      SELECT
        id,
        run_id,
        profile_key,
        crawler_module,
        state_at_start,
        state_at_end,
        total_steps,
        duration_ms,
        success,
        error_message,
        products_found,
        started_at,
        completed_at
      FROM crawl_orchestration_traces
      WHERE dispensary_id = $1
      ORDER BY started_at DESC
      LIMIT $2
    `, [dispensaryId, limitNum]);

    // Get next scheduled crawl if available
    const { rows: scheduleRows } = await pool.query(`
      SELECT
        js.id as schedule_id,
        js.job_name,
        js.enabled,
        js.base_interval_minutes,
        js.jitter_minutes,
        js.next_run_at,
        js.last_run_at,
        js.last_status
      FROM job_schedules js
      WHERE js.enabled = true
        AND js.job_config->>'dispensaryId' = $1::text
      ORDER BY js.next_run_at
      LIMIT 1
    `, [dispensaryId.toString()]);

    // Get dispensary info for slug
    const { rows: dispRows } = await pool.query(`
      SELECT
        id,
        name,
        dba_name,
        slug,
        state,
        city,
        menu_type,
        platform_dispensary_id,
        last_menu_scrape
      FROM dispensaries
      WHERE id = $1
    `, [dispensaryId]);

    res.json({
      dispensary: dispRows[0] || null,
      history: historyRows.map(row => ({
        id: row.id,
        runId: row.run_id,
        profileKey: row.profile_key,
        crawlerModule: row.crawler_module,
        stateAtStart: row.state_at_start,
        stateAtEnd: row.state_at_end,
        totalSteps: row.total_steps,
        durationMs: row.duration_ms,
        success: row.success,
        errorMessage: row.error_message,
        productsFound: row.products_found,
        startedAt: row.started_at?.toISOString() || null,
        completedAt: row.completed_at?.toISOString() || null,
      })),
      nextSchedule: scheduleRows[0] ? {
        scheduleId: scheduleRows[0].schedule_id,
        jobName: scheduleRows[0].job_name,
        enabled: scheduleRows[0].enabled,
        baseIntervalMinutes: scheduleRows[0].base_interval_minutes,
        jitterMinutes: scheduleRows[0].jitter_minutes,
        nextRunAt: scheduleRows[0].next_run_at?.toISOString() || null,
        lastRunAt: scheduleRows[0].last_run_at?.toISOString() || null,
        lastStatus: scheduleRows[0].last_status,
      } : null,
    });
  } catch (error: any) {
    console.error('[Markets] Error fetching crawl history:', error.message);
    res.status(500).json({ error: error.message });
  }
});

/**
 * GET /api/markets/stores/:id/products
 * Get products for a store with filtering and pagination
 */
router.get('/stores/:id/products', async (req: Request, res: Response) => {
  try {
    const { id } = req.params;
    const {
      stockStatus,
      type,
      subcategory,
      brandName,
      search,
      limit = '25',
      offset = '0'
    } = req.query;

    const dispensaryId = parseInt(id, 10);

    let whereClause = 'WHERE sp.dispensary_id = $1';
    const params: any[] = [dispensaryId];
    let paramIndex = 2;

    if (stockStatus) {
      whereClause += ` AND sp.stock_status = $${paramIndex}`;
      params.push(stockStatus);
      paramIndex++;
    }

    if (type) {
      whereClause += ` AND sp.category_raw = $${paramIndex}`;
      params.push(type);
      paramIndex++;
    }

    if (subcategory) {
      whereClause += ` AND sp.subcategory_raw = $${paramIndex}`;
      params.push(subcategory);
      paramIndex++;
    }

    if (brandName) {
      whereClause += ` AND sp.brand_name_raw ILIKE $${paramIndex}`;
      params.push(`%${brandName}%`);
      paramIndex++;
    }

    if (search) {
      whereClause += ` AND (sp.name_raw ILIKE $${paramIndex} OR sp.brand_name_raw ILIKE $${paramIndex})`;
      params.push(`%${search}%`);
      paramIndex++;
    }

    const limitNum = Math.min(parseInt(limit as string, 10), 100);
    const offsetNum = parseInt(offset as string, 10);
    params.push(limitNum, offsetNum);

    // Get products with latest snapshot data using canonical tables
    const { rows } = await pool.query(`
      SELECT
        sp.id,
        sp.external_product_id as external_id,
        sp.name_raw as name,
        sp.brand_name_raw as brand,
        sp.category_raw as type,
        sp.subcategory_raw as subcategory,
        sp.strain_type,
        sp.stock_status,
        sp.stock_status = 'in_stock' as in_stock,
        sp.stock_status != 'missing_from_feed' as is_present_in_feed,
        sp.stock_status = 'missing_from_feed' as missing_from_feed,
        sp.thc_percent as thc_percentage,
        sp.cbd_percent as cbd_percentage,
        sp.primary_image_url as image_url,
        sp.description,
        sp.total_quantity_available as total_quantity,
        sp.first_seen_at,
        sp.last_seen_at,
        sp.updated_at,
        (
          SELECT jsonb_build_object(
            'regular_price', COALESCE(sps.price_rec, 0)::numeric,
            'sale_price', CASE WHEN sps.price_rec_special > 0
                          THEN sps.price_rec_special::numeric
                          ELSE NULL END,
            'med_price', COALESCE(sps.price_med, 0)::numeric,
            'med_sale_price', CASE WHEN sps.price_med_special > 0
                              THEN sps.price_med_special::numeric
                              ELSE NULL END,
            'snapshot_at', sps.captured_at
          )
          FROM store_product_snapshots sps
          WHERE sps.store_product_id = sp.id
          ORDER BY sps.captured_at DESC
          LIMIT 1
        ) as pricing
      FROM store_products sp
      ${whereClause}
      ORDER BY sp.name_raw
      LIMIT $${paramIndex} OFFSET $${paramIndex + 1}
    `, params);

    // Flatten pricing into the product object
    const products = rows.map((row: any) => {
      const pricing = row.pricing || {};
      return {
        ...row,
        regular_price: pricing.regular_price || null,
        sale_price: pricing.sale_price || null,
        med_price: pricing.med_price || null,
        med_sale_price: pricing.med_sale_price || null,
        snapshot_at: pricing.snapshot_at || null,
        pricing: undefined, // Remove the nested object
      };
    });

    // Get total count
    const { rows: countRows } = await pool.query(
      `SELECT COUNT(*) as total FROM store_products sp ${whereClause}`,
      params.slice(0, -2)
    );

    res.json({
      products,
      total: parseInt(countRows[0]?.total || '0', 10),
      limit: limitNum,
      offset: offsetNum,
    });
  } catch (error: any) {
    console.error('[Markets] Error fetching store products:', error.message);
    res.status(500).json({ error: error.message });
  }
});

/**
 * GET /api/markets/stores/:id/brands
 * Get brands for a store
 */
router.get('/stores/:id/brands', async (req: Request, res: Response) => {
  try {
    const { id } = req.params;
    const dispensaryId = parseInt(id, 10);

    const { rows } = await pool.query(`
      SELECT brand_name_raw as brand, COUNT(*) as product_count
      FROM store_products
      WHERE dispensary_id = $1 AND brand_name_raw IS NOT NULL
      GROUP BY brand_name_raw
      ORDER BY product_count DESC, brand_name_raw
    `, [dispensaryId]);

    res.json({ brands: rows });
  } catch (error: any) {
    console.error('[Markets] Error fetching store brands:', error.message);
    res.status(500).json({ error: error.message });
  }
});

/**
 * GET /api/markets/stores/:id/categories
 * Get categories for a store
 */
router.get('/stores/:id/categories', async (req: Request, res: Response) => {
  try {
    const { id } = req.params;
    const dispensaryId = parseInt(id, 10);

    const { rows } = await pool.query(`
      SELECT category_raw as type, subcategory_raw as subcategory, COUNT(*) as product_count
      FROM store_products
      WHERE dispensary_id = $1
      GROUP BY category_raw, subcategory_raw
      ORDER BY product_count DESC
    `, [dispensaryId]);

    res.json({ categories: rows });
  } catch (error: any) {
    console.error('[Markets] Error fetching store categories:', error.message);
    res.status(500).json({ error: error.message });
  }
});

/**
 * POST /api/markets/stores/:id/crawl
 * Trigger a crawl for a store (alias for existing crawl endpoint)
 */
router.post('/stores/:id/crawl', async (req: Request, res: Response) => {
  try {
    const { id } = req.params;
    const dispensaryId = parseInt(id, 10);

    // Verify store exists and has platform_dispensary_id
    const { rows } = await pool.query(`
      SELECT id, name, platform_dispensary_id, menu_type
      FROM dispensaries
      WHERE id = $1
    `, [dispensaryId]);

    if (rows.length === 0) {
      return res.status(404).json({ error: 'Store not found' });
    }

    const store = rows[0];

    if (!store.platform_dispensary_id) {
      return res.status(400).json({
        error: 'Store does not have a platform ID resolved. Cannot crawl.',
        store: { id: store.id, name: store.name, menu_type: store.menu_type }
      });
    }

    // Insert a job into the crawl queue
    await pool.query(`
      INSERT INTO crawl_jobs (dispensary_id, job_type, status, created_at)
      VALUES ($1, 'dutchie_product_crawl', 'pending', NOW())
    `, [dispensaryId]);

    res.json({
      success: true,
      message: `Crawl queued for ${store.name}`,
      store: { id: store.id, name: store.name }
    });
  } catch (error: any) {
    console.error('[Markets] Error triggering crawl:', error.message);
    res.status(500).json({ error: error.message });
  }
});

/**
 * GET /api/markets/brands
 * List all brands with product counts and store presence
 */
router.get('/brands', async (req: Request, res: Response) => {
  try {
    const { search, limit = '100', offset = '0', sortBy = 'products' } = req.query;
    const limitNum = Math.min(parseInt(limit as string, 10), 500);
    const offsetNum = parseInt(offset as string, 10);

    let whereClause = 'WHERE brand_name_raw IS NOT NULL AND brand_name_raw != \'\'';
    const params: any[] = [];
    let paramIndex = 1;

    if (search) {
      whereClause += ` AND brand_name_raw ILIKE $${paramIndex}`;
      params.push(`%${search}%`);
      paramIndex++;
    }

    // Determine sort column
    let orderBy = 'product_count DESC';
    if (sortBy === 'stores') {
      orderBy = 'store_count DESC';
    } else if (sortBy === 'name') {
      orderBy = 'brand_name ASC';
    }

    params.push(limitNum, offsetNum);

    const { rows } = await pool.query(`
      SELECT
        brand_name_raw as brand_name,
        COUNT(*) as product_count,
        COUNT(DISTINCT dispensary_id) as store_count,
        AVG(price_rec) FILTER (WHERE price_rec > 0) as avg_price,
        array_agg(DISTINCT category_raw) FILTER (WHERE category_raw IS NOT NULL) as categories,
        MIN(first_seen_at) as first_seen_at,
        MAX(last_seen_at) as last_seen_at
      FROM store_products
      ${whereClause}
      GROUP BY brand_name_raw
      ORDER BY ${orderBy}
      LIMIT $${paramIndex} OFFSET $${paramIndex + 1}
    `, params);

    // Get total count
    const { rows: countRows } = await pool.query(`
      SELECT COUNT(DISTINCT brand_name_raw) as total
      FROM store_products
      ${whereClause}
    `, params.slice(0, -2));

    // Calculate summary stats
    const { rows: summaryRows } = await pool.query(`
      SELECT
        COUNT(DISTINCT brand_name_raw) as total_brands,
        AVG(product_count) as avg_products_per_brand
      FROM (
        SELECT brand_name_raw, COUNT(*) as product_count
        FROM store_products
        WHERE brand_name_raw IS NOT NULL AND brand_name_raw != ''
        GROUP BY brand_name_raw
      ) brand_counts
    `);

    res.json({
      brands: rows.map((r: any, idx: number) => ({
        id: idx + 1 + offsetNum,
        name: r.brand_name,
        normalized_name: null,
        product_count: parseInt(r.product_count, 10),
        store_count: parseInt(r.store_count, 10),
        avg_price: r.avg_price ? parseFloat(r.avg_price) : null,
        categories: r.categories || [],
        is_portfolio: false,
        first_seen_at: r.first_seen_at,
        last_seen_at: r.last_seen_at,
      })),
      total: parseInt(countRows[0]?.total || '0', 10),
      summary: {
        total_brands: parseInt(summaryRows[0]?.total_brands || '0', 10),
        portfolio_brands: 0,
        avg_products_per_brand: Math.round(parseFloat(summaryRows[0]?.avg_products_per_brand || '0')),
        top_categories: [],
      },
      limit: limitNum,
      offset: offsetNum,
    });
  } catch (error: any) {
    console.error('[Markets] Error fetching brands:', error.message);
    res.status(500).json({ error: error.message });
  }
});

/**
 * GET /api/markets/categories
 * List all categories with product counts
 */
router.get('/categories', async (req: Request, res: Response) => {
  try {
    const { search, limit = '100' } = req.query;
    const limitNum = Math.min(parseInt(limit as string, 10), 500);

    let whereClause = 'WHERE category_raw IS NOT NULL AND category_raw != \'\'';
    const params: any[] = [];
    let paramIndex = 1;

    if (search) {
      whereClause += ` AND category_raw ILIKE $${paramIndex}`;
      params.push(`%${search}%`);
      paramIndex++;
    }

    params.push(limitNum);

    const { rows } = await pool.query(`
      SELECT
        category_raw as name,
        COUNT(*) as product_count,
        COUNT(DISTINCT dispensary_id) as store_count,
        AVG(price_rec) FILTER (WHERE price_rec > 0) as avg_price
      FROM store_products
      ${whereClause}
      GROUP BY category_raw
      ORDER BY product_count DESC
      LIMIT $${paramIndex}
    `, params);

    res.json({
      categories: rows.map((r: any, idx: number) => ({
        id: idx + 1,
        name: r.name,
        product_count: parseInt(r.product_count, 10),
        store_count: parseInt(r.store_count, 10),
        avg_price: r.avg_price ? parseFloat(r.avg_price) : null,
      })),
      total: rows.length,
    });
  } catch (error: any) {
    console.error('[Markets] Error fetching categories:', error.message);
    res.status(500).json({ error: error.message });
  }
});

export default router;