From 9d8972aa86afd2d009705fda2f9103f3cfe5fc91 Mon Sep 17 00:00:00 2001 From: Kelly Date: Mon, 1 Dec 2025 00:07:00 -0700 Subject: [PATCH] Fix category-crawler-jobs store lookup query MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Fix column name from s.dutchie_plus_url to s.dutchie_url - Add availability tracking and product freshness APIs - Add crawl script for sequential dispensary processing 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- .../024_product_availability_tracking.sql | 52 + backend/src/routes/products.ts | 73 + backend/src/routes/stores.ts | 138 +- backend/src/scripts/crawl-five-sequential.ts | 26 + backend/src/services/availability.ts | 240 ++ backend/src/services/category-crawler-jobs.ts | 4 +- backend/src/services/crawler-jobs.ts | 5 + backend/src/services/scraper.ts | 66 +- docs/CANNABRANDS_API_FRONTEND_SPEC.md | 1243 +++++++ docs/CRAWL_OPERATIONS.md | 592 ++++ ...PRODUCT_BRAND_INTELLIGENCE_ARCHITECTURE.md | 2004 +++++++++++ docs/PRODUCT_BRAND_INTELLIGENCE_FINAL.md | 2027 +++++++++++ docs/STORE_API_SPECIFICATION.md | 1958 +++++++++++ docs/WORDPRESS_PLUGIN_SPEC.md | 2994 +++++++++++++++++ frontend/src/pages/StoreDetail.tsx | 224 +- 15 files changed, 11604 insertions(+), 42 deletions(-) create mode 100644 backend/migrations/024_product_availability_tracking.sql create mode 100644 backend/src/scripts/crawl-five-sequential.ts create mode 100644 backend/src/services/availability.ts create mode 100644 docs/CANNABRANDS_API_FRONTEND_SPEC.md create mode 100644 docs/CRAWL_OPERATIONS.md create mode 100644 docs/PRODUCT_BRAND_INTELLIGENCE_ARCHITECTURE.md create mode 100644 docs/PRODUCT_BRAND_INTELLIGENCE_FINAL.md create mode 100644 docs/STORE_API_SPECIFICATION.md create mode 100644 docs/WORDPRESS_PLUGIN_SPEC.md diff --git a/backend/migrations/024_product_availability_tracking.sql b/backend/migrations/024_product_availability_tracking.sql new file mode 100644 index 00000000..14017ede --- /dev/null +++ b/backend/migrations/024_product_availability_tracking.sql @@ -0,0 +1,52 @@ +-- Migration 024: Product Availability Tracking +-- Adds normalized availability status and transition tracking + +-- Add availability columns to products table +ALTER TABLE products ADD COLUMN IF NOT EXISTS availability_status VARCHAR(20) DEFAULT 'unknown'; +ALTER TABLE products ADD COLUMN IF NOT EXISTS availability_raw JSONB; +ALTER TABLE products ADD COLUMN IF NOT EXISTS last_seen_in_stock_at TIMESTAMPTZ; +ALTER TABLE products ADD COLUMN IF NOT EXISTS last_seen_out_of_stock_at TIMESTAMPTZ; + +-- Add comment for clarity +COMMENT ON COLUMN products.availability_status IS 'Normalized status: in_stock, out_of_stock, limited, unknown'; +COMMENT ON COLUMN products.availability_raw IS 'Raw availability payload from provider for debugging'; +COMMENT ON COLUMN products.last_seen_in_stock_at IS 'Last time product was seen in stock'; +COMMENT ON COLUMN products.last_seen_out_of_stock_at IS 'Last time product was seen out of stock'; + +-- Create indexes for availability queries +CREATE INDEX IF NOT EXISTS idx_products_availability_status ON products(availability_status); +CREATE INDEX IF NOT EXISTS idx_products_availability_by_store ON products(store_id, availability_status) WHERE store_id IS NOT NULL; +CREATE INDEX IF NOT EXISTS idx_products_availability_by_dispensary ON products(dispensary_id, availability_status) WHERE dispensary_id IS NOT NULL; + +-- Backfill availability_status from existing in_stock column +UPDATE products +SET availability_status = CASE + WHEN in_stock = true THEN 'in_stock' + WHEN in_stock = false THEN 'out_of_stock' + ELSE 'unknown' +END +WHERE availability_status = 'unknown' OR availability_status IS NULL; + +-- Set last_seen_in_stock_at for currently in-stock products +UPDATE products +SET last_seen_in_stock_at = COALESCE(last_seen_at, updated_at, NOW()) +WHERE in_stock = true AND last_seen_in_stock_at IS NULL; + +-- Set last_seen_out_of_stock_at for currently out-of-stock products +UPDATE products +SET last_seen_out_of_stock_at = COALESCE(last_seen_at, updated_at, NOW()) +WHERE in_stock = false AND last_seen_out_of_stock_at IS NULL; + +-- Add availability tracking to dispensary_crawl_jobs +ALTER TABLE dispensary_crawl_jobs ADD COLUMN IF NOT EXISTS in_stock_count INTEGER; +ALTER TABLE dispensary_crawl_jobs ADD COLUMN IF NOT EXISTS out_of_stock_count INTEGER; +ALTER TABLE dispensary_crawl_jobs ADD COLUMN IF NOT EXISTS limited_count INTEGER; +ALTER TABLE dispensary_crawl_jobs ADD COLUMN IF NOT EXISTS unknown_count INTEGER; +ALTER TABLE dispensary_crawl_jobs ADD COLUMN IF NOT EXISTS availability_changed_count INTEGER; + +-- Add availability tracking to crawl_jobs (store-based) +ALTER TABLE crawl_jobs ADD COLUMN IF NOT EXISTS in_stock_count INTEGER; +ALTER TABLE crawl_jobs ADD COLUMN IF NOT EXISTS out_of_stock_count INTEGER; +ALTER TABLE crawl_jobs ADD COLUMN IF NOT EXISTS limited_count INTEGER; +ALTER TABLE crawl_jobs ADD COLUMN IF NOT EXISTS unknown_count INTEGER; +ALTER TABLE crawl_jobs ADD COLUMN IF NOT EXISTS availability_changed_count INTEGER; diff --git a/backend/src/routes/products.ts b/backend/src/routes/products.ts index 9a441fcd..298cbd65 100755 --- a/backend/src/routes/products.ts +++ b/backend/src/routes/products.ts @@ -6,6 +6,55 @@ import { getImageUrl } from '../utils/minio'; const router = Router(); router.use(authMiddleware); +// Freshness threshold: data older than this is considered stale +const STALE_THRESHOLD_HOURS = 4; + +interface FreshnessInfo { + last_crawl_at: string | null; + is_stale: boolean; + freshness: string; + hours_since_crawl: number | null; +} + +function calculateFreshness(lastCrawlAt: Date | null): FreshnessInfo { + if (!lastCrawlAt) { + return { + last_crawl_at: null, + is_stale: true, + freshness: 'Never crawled', + hours_since_crawl: null + }; + } + + const now = new Date(); + const diffMs = now.getTime() - lastCrawlAt.getTime(); + const diffHours = diffMs / (1000 * 60 * 60); + const isStale = diffHours > STALE_THRESHOLD_HOURS; + + let freshnessText: string; + if (diffHours < 1) { + const mins = Math.round(diffHours * 60); + freshnessText = `Last crawled ${mins} minute${mins !== 1 ? 's' : ''} ago`; + } else if (diffHours < 24) { + const hrs = Math.round(diffHours); + freshnessText = `Last crawled ${hrs} hour${hrs !== 1 ? 's' : ''} ago`; + } else { + const days = Math.round(diffHours / 24); + freshnessText = `Last crawled ${days} day${days !== 1 ? 's' : ''} ago`; + } + + if (isStale) { + freshnessText += ' (STALE)'; + } + + return { + last_crawl_at: lastCrawlAt.toISOString(), + is_stale: isStale, + freshness: freshnessText, + hours_since_crawl: Math.round(diffHours * 10) / 10 + }; +} + // Helper function to filter fields from object function selectFields(obj: any, fields: string[]): any { if (!fields || fields.length === 0) return obj; @@ -216,11 +265,35 @@ router.get('/', async (req, res) => { const countResult = await pool.query(countQuery, countParams); + // Get freshness info if store_id is specified + let freshnessInfo: FreshnessInfo | null = null; + let storeInfo: { id: number; name: string } | null = null; + + if (store_id) { + const storeResult = await pool.query( + 'SELECT id, name, last_scraped_at FROM stores WHERE id = $1', + [store_id] + ); + if (storeResult.rows.length > 0) { + const store = storeResult.rows[0]; + storeInfo = { id: store.id, name: store.name }; + freshnessInfo = calculateFreshness(store.last_scraped_at); + } + } + res.json({ products, total: parseInt(countResult.rows[0].count), limit: parseInt(limit as string), offset: parseInt(offset as string), + // Add freshness metadata when store_id is provided + ...(freshnessInfo && { + store: storeInfo, + last_crawl_at: freshnessInfo.last_crawl_at, + is_stale: freshnessInfo.is_stale, + freshness: freshnessInfo.freshness, + hours_since_crawl: freshnessInfo.hours_since_crawl + }), filters: { store_id, category_id, diff --git a/backend/src/routes/stores.ts b/backend/src/routes/stores.ts index 7a231119..4ce4439a 100755 --- a/backend/src/routes/stores.ts +++ b/backend/src/routes/stores.ts @@ -28,28 +28,150 @@ router.get('/', async (req, res) => { } }); -// Get single store +// Freshness threshold in hours +const STALE_THRESHOLD_HOURS = 4; + +function calculateFreshness(lastScrapedAt: Date | null): { + last_scraped_at: string | null; + is_stale: boolean; + freshness: string; + hours_since_scrape: number | null; +} { + if (!lastScrapedAt) { + return { + last_scraped_at: null, + is_stale: true, + freshness: 'Never scraped', + hours_since_scrape: null + }; + } + + const now = new Date(); + const diffMs = now.getTime() - lastScrapedAt.getTime(); + const diffHours = diffMs / (1000 * 60 * 60); + const isStale = diffHours > STALE_THRESHOLD_HOURS; + + let freshnessText: string; + if (diffHours < 1) { + const mins = Math.round(diffHours * 60); + freshnessText = `${mins} minute${mins !== 1 ? 's' : ''} ago`; + } else if (diffHours < 24) { + const hrs = Math.round(diffHours); + freshnessText = `${hrs} hour${hrs !== 1 ? 's' : ''} ago`; + } else { + const days = Math.round(diffHours / 24); + freshnessText = `${days} day${days !== 1 ? 's' : ''} ago`; + } + + return { + last_scraped_at: lastScrapedAt.toISOString(), + is_stale: isStale, + freshness: freshnessText, + hours_since_scrape: Math.round(diffHours * 10) / 10 + }; +} + +function detectProvider(dutchieUrl: string | null): string { + if (!dutchieUrl) return 'unknown'; + if (dutchieUrl.includes('dutchie.com')) return 'Dutchie'; + if (dutchieUrl.includes('iheartjane.com') || dutchieUrl.includes('jane.co')) return 'Jane'; + if (dutchieUrl.includes('treez.io')) return 'Treez'; + if (dutchieUrl.includes('weedmaps.com')) return 'Weedmaps'; + if (dutchieUrl.includes('leafly.com')) return 'Leafly'; + return 'Custom'; +} + +// Get single store with full details router.get('/:id', async (req, res) => { try { const { id } = req.params; - + + // Get store with counts and linked dispensary const result = await pool.query(` - SELECT + SELECT s.*, + d.id as dispensary_id, + d.name as dispensary_name, + d.slug as dispensary_slug, + d.state as dispensary_state, + d.city as dispensary_city, + d.address as dispensary_address, + d.menu_provider as dispensary_menu_provider, COUNT(DISTINCT p.id) as product_count, - COUNT(DISTINCT c.id) as category_count + COUNT(DISTINCT c.id) as category_count, + COUNT(DISTINCT p.id) FILTER (WHERE p.in_stock = true) as in_stock_count, + COUNT(DISTINCT p.id) FILTER (WHERE p.in_stock = false) as out_of_stock_count FROM stores s + LEFT JOIN dispensaries d ON s.dispensary_id = d.id LEFT JOIN products p ON s.id = p.store_id LEFT JOIN categories c ON s.id = c.store_id WHERE s.id = $1 - GROUP BY s.id + GROUP BY s.id, d.id, d.name, d.slug, d.state, d.city, d.address, d.menu_provider `, [id]); - + if (result.rows.length === 0) { return res.status(404).json({ error: 'Store not found' }); } - - res.json(result.rows[0]); + + const store = result.rows[0]; + + // Get recent crawl jobs for this store + const jobsResult = await pool.query(` + SELECT + id, status, job_type, trigger_type, + started_at, completed_at, + products_found, products_new, products_updated, + in_stock_count, out_of_stock_count, + error_message + FROM crawl_jobs + WHERE store_id = $1 + ORDER BY created_at DESC + LIMIT 10 + `, [id]); + + // Get schedule info if exists + const scheduleResult = await pool.query(` + SELECT + enabled, interval_hours, next_run_at, last_run_at + FROM store_crawl_schedule + WHERE store_id = $1 + `, [id]); + + // Calculate freshness + const freshness = calculateFreshness(store.last_scraped_at); + + // Detect provider from URL + const provider = detectProvider(store.dutchie_url); + + // Build response + const response = { + ...store, + provider, + freshness: freshness.freshness, + is_stale: freshness.is_stale, + hours_since_scrape: freshness.hours_since_scrape, + linked_dispensary: store.dispensary_id ? { + id: store.dispensary_id, + name: store.dispensary_name, + slug: store.dispensary_slug, + state: store.dispensary_state, + city: store.dispensary_city, + address: store.dispensary_address, + menu_provider: store.dispensary_menu_provider + } : null, + schedule: scheduleResult.rows[0] || null, + recent_jobs: jobsResult.rows + }; + + // Remove redundant dispensary fields from root + delete response.dispensary_name; + delete response.dispensary_slug; + delete response.dispensary_state; + delete response.dispensary_city; + delete response.dispensary_address; + delete response.dispensary_menu_provider; + + res.json(response); } catch (error) { console.error('Error fetching store:', error); res.status(500).json({ error: 'Failed to fetch store' }); diff --git a/backend/src/scripts/crawl-five-sequential.ts b/backend/src/scripts/crawl-five-sequential.ts new file mode 100644 index 00000000..aa6138b2 --- /dev/null +++ b/backend/src/scripts/crawl-five-sequential.ts @@ -0,0 +1,26 @@ +import { runDispensaryOrchestrator } from '../services/dispensary-orchestrator'; + +// Run 5 crawlers sequentially to avoid OOM +const dispensaryIds = [112, 81, 115, 140, 177]; + +async function run() { + console.log('Starting 5 crawlers SEQUENTIALLY...'); + + for (const id of dispensaryIds) { + console.log(`\n=== Starting crawler for dispensary ${id} ===`); + try { + const result = await runDispensaryOrchestrator(id); + console.log(` Status: ${result.status}`); + console.log(` Summary: ${result.summary}`); + if (result.productsFound) { + console.log(` Products: ${result.productsFound} found, ${result.productsNew} new, ${result.productsUpdated} updated`); + } + } catch (e: any) { + console.log(` ERROR: ${e.message}`); + } + } + + console.log('\n=== All 5 crawlers complete ==='); +} + +run().catch(e => console.log('Fatal:', e.message)); diff --git a/backend/src/services/availability.ts b/backend/src/services/availability.ts new file mode 100644 index 00000000..292937f0 --- /dev/null +++ b/backend/src/services/availability.ts @@ -0,0 +1,240 @@ +/** + * Availability Service + * + * Normalizes product availability from various menu providers and tracks + * state transitions for inventory analytics. + */ + +// Threshold for considering stock as "limited" +const LIMITED_THRESHOLD = 5; + +export type AvailabilityStatus = 'in_stock' | 'out_of_stock' | 'limited' | 'unknown'; + +export interface NormalizedAvailability { + status: AvailabilityStatus; + quantity: number | null; + raw: any; +} + +export interface AvailabilityHints { + hasOutOfStockBadge?: boolean; + hasLimitedBadge?: boolean; + hasInStockBadge?: boolean; + stockText?: string; + quantityText?: string; +} + +/** + * Normalize availability from a Dutchie product + * + * Dutchie products can have various availability indicators: + * - potencyAmount.quantity: explicit stock count + * - status: sometimes includes stock status + * - variants[].quantity: stock per variant + * - isInStock / inStock: boolean flags + */ +export function normalizeAvailability(dutchieProduct: any): NormalizedAvailability { + const raw: any = {}; + + // Collect raw availability data for debugging + if (dutchieProduct.potencyAmount?.quantity !== undefined) { + raw.potencyQuantity = dutchieProduct.potencyAmount.quantity; + } + if (dutchieProduct.status !== undefined) { + raw.status = dutchieProduct.status; + } + if (dutchieProduct.isInStock !== undefined) { + raw.isInStock = dutchieProduct.isInStock; + } + if (dutchieProduct.inStock !== undefined) { + raw.inStock = dutchieProduct.inStock; + } + if (dutchieProduct.variants?.length) { + const variantQuantities = dutchieProduct.variants + .filter((v: any) => v.quantity !== undefined) + .map((v: any) => ({ option: v.option, quantity: v.quantity })); + if (variantQuantities.length) { + raw.variantQuantities = variantQuantities; + } + } + + // Try to extract quantity + let quantity: number | null = null; + + // Check potencyAmount.quantity first (most reliable for Dutchie) + if (typeof dutchieProduct.potencyAmount?.quantity === 'number') { + quantity = dutchieProduct.potencyAmount.quantity; + } + // Sum variant quantities if available + else if (dutchieProduct.variants?.length) { + const totalVariantQty = dutchieProduct.variants.reduce((sum: number, v: any) => { + return sum + (typeof v.quantity === 'number' ? v.quantity : 0); + }, 0); + if (totalVariantQty > 0) { + quantity = totalVariantQty; + } + } + + // Determine status + let status: AvailabilityStatus = 'unknown'; + + // Explicit boolean flags take precedence + if (dutchieProduct.isInStock === false || dutchieProduct.inStock === false) { + status = 'out_of_stock'; + } else if (dutchieProduct.isInStock === true || dutchieProduct.inStock === true) { + status = quantity !== null && quantity <= LIMITED_THRESHOLD ? 'limited' : 'in_stock'; + } + // Check status string + else if (typeof dutchieProduct.status === 'string') { + const statusLower = dutchieProduct.status.toLowerCase(); + if (statusLower.includes('out') || statusLower.includes('unavailable')) { + status = 'out_of_stock'; + } else if (statusLower.includes('limited') || statusLower.includes('low')) { + status = 'limited'; + } else if (statusLower.includes('in') || statusLower.includes('available')) { + status = 'in_stock'; + } + } + // Infer from quantity + else if (quantity !== null) { + if (quantity === 0) { + status = 'out_of_stock'; + } else if (quantity <= LIMITED_THRESHOLD) { + status = 'limited'; + } else { + status = 'in_stock'; + } + } + + return { status, quantity, raw }; +} + +/** + * Extract availability hints from page content or product card HTML + * + * Used for sandbox provider scraping where we don't have structured data + */ +export function extractAvailabilityHints(pageContent: string, productElement?: string): AvailabilityHints { + const hints: AvailabilityHints = {}; + const content = (productElement || pageContent).toLowerCase(); + + // Check for out-of-stock indicators + const oosPatterns = [ + 'out of stock', + 'out-of-stock', + 'sold out', + 'soldout', + 'unavailable', + 'not available', + 'coming soon', + 'notify me' + ]; + hints.hasOutOfStockBadge = oosPatterns.some(p => content.includes(p)); + + // Check for limited stock indicators + const limitedPatterns = [ + 'limited stock', + 'limited quantity', + 'low stock', + 'only \\d+ left', + 'few remaining', + 'almost gone', + 'selling fast' + ]; + hints.hasLimitedBadge = limitedPatterns.some(p => { + if (p.includes('\\d')) { + return new RegExp(p, 'i').test(content); + } + return content.includes(p); + }); + + // Check for in-stock indicators + const inStockPatterns = [ + 'in stock', + 'in-stock', + 'add to cart', + 'add to bag', + 'buy now', + 'available' + ]; + hints.hasInStockBadge = inStockPatterns.some(p => content.includes(p)); + + // Try to extract quantity text + const qtyMatch = content.match(/(\d+)\s*(left|remaining|in stock|available)/i); + if (qtyMatch) { + hints.quantityText = qtyMatch[0]; + } + + // Look for explicit stock text + const stockTextMatch = content.match(/(out of stock|in stock|low stock|limited|sold out)[^<]*/i); + if (stockTextMatch) { + hints.stockText = stockTextMatch[0].trim(); + } + + return hints; +} + +/** + * Convert availability hints to normalized availability + */ +export function hintsToAvailability(hints: AvailabilityHints): NormalizedAvailability { + let status: AvailabilityStatus = 'unknown'; + let quantity: number | null = null; + + // Extract quantity if present + if (hints.quantityText) { + const match = hints.quantityText.match(/(\d+)/); + if (match) { + quantity = parseInt(match[1], 10); + } + } + + // Determine status from hints + if (hints.hasOutOfStockBadge) { + status = 'out_of_stock'; + } else if (hints.hasLimitedBadge) { + status = 'limited'; + } else if (hints.hasInStockBadge) { + status = quantity !== null && quantity <= LIMITED_THRESHOLD ? 'limited' : 'in_stock'; + } + + return { + status, + quantity, + raw: hints + }; +} + +/** + * Aggregate availability counts from a list of products + */ +export interface AvailabilityCounts { + in_stock: number; + out_of_stock: number; + limited: number; + unknown: number; + changed: number; +} + +export function aggregateAvailability( + products: Array<{ availability_status?: AvailabilityStatus; previous_status?: AvailabilityStatus }> +): AvailabilityCounts { + const counts: AvailabilityCounts = { + in_stock: 0, + out_of_stock: 0, + limited: 0, + unknown: 0, + changed: 0 + }; + + for (const product of products) { + const status = product.availability_status || 'unknown'; + counts[status]++; + + if (product.previous_status && product.previous_status !== status) { + counts.changed++; + } + } + + return counts; +} diff --git a/backend/src/services/category-crawler-jobs.ts b/backend/src/services/category-crawler-jobs.ts index 426bef0e..5935dc6d 100644 --- a/backend/src/services/category-crawler-jobs.ts +++ b/backend/src/services/category-crawler-jobs.ts @@ -106,9 +106,10 @@ async function updateCategoryScanTime( } async function getStoreIdForDispensary(dispensaryId: number): Promise { + // First check if dispensary has menu_url - if so, try to match with stores.dutchie_url const result = await pool.query( `SELECT s.id FROM stores s - JOIN dispensaries d ON d.menu_url = s.dutchie_plus_url OR d.name ILIKE '%' || s.name || '%' + JOIN dispensaries d ON d.menu_url = s.dutchie_url OR d.name ILIKE '%' || s.name || '%' WHERE d.id = $1 LIMIT 1`, [dispensaryId] @@ -118,6 +119,7 @@ async function getStoreIdForDispensary(dispensaryId: number): Promise { const sanitized = sanitizeProductData(p); + // Normalize availability from Dutchie product data + const availability = normalizeAvailability(p); return { dutchieProductId: `${category.store_slug}-${category.slug}-${Date.now()}-${index}`, @@ -599,7 +606,10 @@ export async function scrapeCategory(storeId: number, categoryId: number, userAg weight: sanitized.weight, imageUrl: p.imageUrl, dutchieUrl: p.href, - metadata: p.metadata || {} + metadata: p.metadata || {}, + availabilityStatus: availability.status, + availabilityRaw: availability.raw, + stockQuantity: availability.quantity }; }); @@ -660,47 +670,72 @@ async function autoScroll(page: Page) { export async function saveProducts(storeId: number, categoryId: number, products: Product[]): Promise { const client = await pool.connect(); - + try { await client.query('BEGIN'); - + logger.info('scraper', `Saving ${products.length} products to database...`); - + + // Mark all products as out-of-stock before processing (they'll be re-marked if found) + // Also update availability_status and last_seen_out_of_stock_at for state transition tracking await client.query(` UPDATE products - SET in_stock = false - WHERE store_id = $1 AND category_id = $2 + SET in_stock = false, + availability_status = 'out_of_stock', + last_seen_out_of_stock_at = CASE + WHEN availability_status != 'out_of_stock' THEN CURRENT_TIMESTAMP + ELSE last_seen_out_of_stock_at + END + WHERE store_id = $1 AND category_id = $2 AND in_stock = true `, [storeId, categoryId]); for (const product of products) { try { + // Get availability from product (defaults to in_stock if product exists in scraped data) + const availStatus = product.availabilityStatus || 'in_stock'; + const availRaw = product.availabilityRaw ? JSON.stringify(product.availabilityRaw) : null; + const stockQty = product.stockQuantity ?? null; + const existingResult = await client.query(` - SELECT id, image_url, local_image_path + SELECT id, image_url, local_image_path, availability_status FROM products WHERE store_id = $1 AND name = $2 AND category_id = $3 AND (variant = $4 OR (variant IS NULL AND $4 IS NULL)) `, [storeId, product.name, categoryId, product.variant || null]); - + let localImagePath = null; let productId: number; - + if (existingResult.rows.length > 0) { productId = existingResult.rows[0].id; localImagePath = existingResult.rows[0].local_image_path; - + const prevStatus = existingResult.rows[0].availability_status; + + // Determine if we need to update last_seen_in_stock_at + const isNowInStock = availStatus === 'in_stock' || availStatus === 'limited'; + const wasOutOfStock = prevStatus === 'out_of_stock' || prevStatus === 'unknown'; + await client.query(` UPDATE products SET name = $1, variant = $2, description = $3, price = $4, strain_type = $5, thc_percentage = $6, cbd_percentage = $7, brand = $8, weight = $9, image_url = $10, dutchie_url = $11, in_stock = true, metadata = $12, last_seen_at = CURRENT_TIMESTAMP, - updated_at = CURRENT_TIMESTAMP + updated_at = CURRENT_TIMESTAMP, + availability_status = $14, + availability_raw = $15, + stock_quantity = $16, + last_seen_in_stock_at = CASE + WHEN $17 THEN CURRENT_TIMESTAMP + ELSE last_seen_in_stock_at + END WHERE id = $13 `, [ product.name, product.variant, product.description, product.price, product.strainType, product.thcPercentage, product.cbdPercentage, product.brand, product.weight, product.imageUrl, product.dutchieUrl, - JSON.stringify(product.metadata), productId + JSON.stringify(product.metadata), productId, availStatus, availRaw, stockQty, + isNowInStock && wasOutOfStock ]); } else { // Generate unique slug from product name + timestamp + random suffix @@ -716,14 +751,15 @@ export async function saveProducts(storeId: number, categoryId: number, products INSERT INTO products ( store_id, category_id, dutchie_product_id, name, slug, variant, description, price, strain_type, thc_percentage, cbd_percentage, - brand, weight, image_url, dutchie_url, in_stock, metadata - ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, true, $16) + brand, weight, image_url, dutchie_url, in_stock, metadata, + availability_status, availability_raw, stock_quantity, last_seen_in_stock_at + ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, true, $16, $17, $18, $19, CURRENT_TIMESTAMP) RETURNING id `, [ storeId, categoryId, product.dutchieProductId, product.name, slug, product.variant, product.description, product.price, product.strainType, product.thcPercentage, product.cbdPercentage, product.brand, product.weight, product.imageUrl, product.dutchieUrl, - JSON.stringify(product.metadata) + JSON.stringify(product.metadata), availStatus, availRaw, stockQty ]); productId = insertResult.rows[0].id; diff --git a/docs/CANNABRANDS_API_FRONTEND_SPEC.md b/docs/CANNABRANDS_API_FRONTEND_SPEC.md new file mode 100644 index 00000000..fbc2a742 --- /dev/null +++ b/docs/CANNABRANDS_API_FRONTEND_SPEC.md @@ -0,0 +1,1243 @@ +# Cannabrands Dashboard API - Front-End Ready Specification + +## Overview + +This document defines the **front-end-ready API responses** for the Cannabrands brand dashboard. All responses are optimized for direct consumption by dashboard cards, widgets, and chart components. + +**Base URL:** `https://api.cannabrands.app/v1` + +**Authentication:** Bearer token (JWT) with `brand_key` claim + +--- + +## 1. Brand Key Resolution Flow + +### Resolution Steps + +``` +API Request: GET /v1/brands/cb_12345/dashboard/summary + ↓ + brand_key = "cb_12345" + ↓ + ┌───────────────┴───────────────┐ + │ 1. JWT Validation │ + │ Extract brand_key from │ + │ token claims │ + │ Verify: token.brand_key │ + │ === "cb_12345" │ + └───────────────┬───────────────┘ + ↓ + ┌───────────────┴───────────────┐ + │ 2. Brand Resolution │ + │ Step A: Try cannabrands_id│ + │ SELECT id FROM │ + │ canonical_brands WHERE │ + │ cannabrands_id = │ + │ 'cb_12345' │ + │ │ + │ Step B (fallback): Try │ + │ cannabrands_slug │ + │ SELECT id FROM │ + │ canonical_brands WHERE │ + │ cannabrands_slug = │ + │ 'cb_12345' │ + └───────────────┬───────────────┘ + ↓ + canonical_brand_id = 42 + ↓ + ┌───────────────┴───────────────┐ + │ 3. All Queries Scoped │ + │ WHERE canonical_brand_id │ + │ = 42 │ + └───────────────────────────────┘ +``` + +### TypeScript Implementation + +```typescript +async function resolveBrandKey(db: Pool, brandKey: string): Promise { + // Step A: Try cannabrands_id (primary identifier) + const byId = await db.query( + `SELECT id FROM canonical_brands WHERE cannabrands_id = $1`, + [brandKey] + ); + if (byId.rows.length > 0) { + return byId.rows[0].id; + } + + // Step B: Fallback to cannabrands_slug + const bySlug = await db.query( + `SELECT id FROM canonical_brands WHERE cannabrands_slug = $1`, + [brandKey] + ); + if (bySlug.rows.length > 0) { + return bySlug.rows[0].id; + } + + return null; +} +``` + +--- + +## 2. Dashboard Summary Endpoint + +### `GET /v1/brands/:brand_key/dashboard/summary` + +Returns all KPIs needed for the main dashboard cards in a single request. + +**Query Parameters:** + +| Param | Type | Required | Default | Description | +|-------|------|----------|---------|-------------| +| `window` | string | No | `30d` | Comparison window: `7d`, `30d`, `90d` | + +**Response (Success - 200):** + +```json +{ + "brand_key": "cb_12345", + "brand_name": "Raw Garden", + "as_of": "2025-01-15T08:00:00.000Z", + "window": "30d", + + "store_footprint": { + "current": 127, + "previous": 112, + "delta": 15, + "delta_percent": 13.39, + "direction": "up" + }, + + "active_skus": { + "current": 543, + "previous": 476, + "delta": 67, + "delta_percent": 14.08, + "direction": "up" + }, + + "in_stock_rate": { + "current": 90.1, + "previous": 88.7, + "delta": 1.4, + "direction": "up" + }, + + "promo_exposure": { + "skus_on_promo": 34, + "stores_running_promos": 23, + "percent_of_skus": 6.3, + "direction": "stable" + }, + + "price_summary": { + "avg_cents": 4250, + "min_cents": 2500, + "max_cents": 8500, + "avg_formatted": "$42.50", + "min_formatted": "$25.00", + "max_formatted": "$85.00" + }, + + "activity_last_7d": { + "stores_added": 3, + "stores_removed": 0, + "new_skus": 12, + "removed_skus": 2, + "price_changes": 8 + } +} +``` + +**Dashboard Card Mapping:** + +| Card | Fields | +|------|--------| +| "Store Distribution" | `store_footprint.current`, `store_footprint.delta`, `store_footprint.direction` | +| "Active SKUs" | `active_skus.current`, `active_skus.delta`, `active_skus.direction` | +| "In-Stock Rate" | `in_stock_rate.current` + "%" suffix, `in_stock_rate.direction` | +| "Promotional Activity" | `promo_exposure.skus_on_promo`, `promo_exposure.stores_running_promos` | +| "Avg Price" | `price_summary.avg_formatted` | +| "Recent Activity" | `activity_last_7d.*` | + +**Response (Empty State - 200):** + +```json +{ + "brand_key": "cb_99999", + "brand_name": "New Brand", + "as_of": "2025-01-15T08:00:00.000Z", + "window": "30d", + + "store_footprint": { + "current": 0, + "previous": 0, + "delta": 0, + "delta_percent": 0, + "direction": "stable" + }, + + "active_skus": { + "current": 0, + "previous": 0, + "delta": 0, + "delta_percent": 0, + "direction": "stable" + }, + + "in_stock_rate": { + "current": null, + "previous": null, + "delta": null, + "direction": "stable" + }, + + "promo_exposure": { + "skus_on_promo": 0, + "stores_running_promos": 0, + "percent_of_skus": 0, + "direction": "stable" + }, + + "price_summary": { + "avg_cents": null, + "min_cents": null, + "max_cents": null, + "avg_formatted": null, + "min_formatted": null, + "max_formatted": null + }, + + "activity_last_7d": { + "stores_added": 0, + "stores_removed": 0, + "new_skus": 0, + "removed_skus": 0, + "price_changes": 0 + } +} +``` + +**Direction Values:** + +| Value | Meaning | UI Treatment | +|-------|---------|--------------| +| `"up"` | Increased from previous period | Green arrow, positive indicator | +| `"down"` | Decreased from previous period | Red arrow, negative indicator | +| `"stable"` | No significant change | Gray dash, neutral indicator | + +--- + +## 3. Store Timeseries Endpoint + +### `GET /v1/brands/:brand_key/dashboard/store-timeseries` + +Returns daily store/SKU data points optimized for line/area charts. + +**Query Parameters:** + +| Param | Type | Required | Default | Description | +|-------|------|----------|---------|-------------| +| `from` | string | No | 30 days ago | ISO date `YYYY-MM-DD` | +| `to` | string | No | today | ISO date `YYYY-MM-DD` | +| `interval` | string | No | `day` | Aggregation: `day`, `week`, `month` | + +**Response (Success - 200):** + +```json +{ + "brand_key": "cb_12345", + "period": { + "from": "2024-12-16", + "to": "2025-01-15", + "interval": "day", + "points_count": 31 + }, + + "points": [ + { + "date": "2024-12-16", + "stores": 112, + "stores_added": 2, + "stores_removed": 0, + "stores_net": 2, + "skus": 476, + "skus_in_stock": 423, + "skus_new": 5, + "skus_removed": 1, + "skus_net": 4 + }, + { + "date": "2024-12-17", + "stores": 114, + "stores_added": 2, + "stores_removed": 0, + "stores_net": 2, + "skus": 482, + "skus_in_stock": 430, + "skus_new": 8, + "skus_removed": 2, + "skus_net": 6 + } + ], + + "summary": { + "stores_start": 112, + "stores_end": 127, + "stores_peak": 128, + "stores_low": 112, + "skus_start": 476, + "skus_end": 543, + "skus_peak": 545, + "skus_low": 476 + } +} +``` + +**Chart Component Usage:** + +```typescript +// Example: Recharts line chart +const storeChartData = response.points.map(p => ({ + date: p.date, + stores: p.stores, + skus: p.skus +})); + + + + + +``` + +**Response (Empty State - 200):** + +```json +{ + "brand_key": "cb_99999", + "period": { + "from": "2024-12-16", + "to": "2025-01-15", + "interval": "day", + "points_count": 0 + }, + "points": [], + "summary": { + "stores_start": null, + "stores_end": null, + "stores_peak": null, + "stores_low": null, + "skus_start": null, + "skus_end": null, + "skus_peak": null, + "skus_low": null + } +} +``` + +--- + +## 4. Promo Timeseries Endpoint + +### `GET /v1/brands/:brand_key/dashboard/promo-timeseries` + +Returns daily promotional activity data points for stacked bar/area charts. + +**Query Parameters:** + +| Param | Type | Required | Default | Description | +|-------|------|----------|---------|-------------| +| `from` | string | No | 30 days ago | ISO date `YYYY-MM-DD` | +| `to` | string | No | today | ISO date `YYYY-MM-DD` | +| `interval` | string | No | `day` | Aggregation: `day`, `week`, `month` | + +**Response (Success - 200):** + +```json +{ + "brand_key": "cb_12345", + "period": { + "from": "2024-12-16", + "to": "2025-01-15", + "interval": "day", + "points_count": 31 + }, + + "points": [ + { + "date": "2024-12-16", + "total_promos": 45, + "percent_off": 28, + "dollar_off": 5, + "bogo": 8, + "bundle": 2, + "set_price": 1, + "other": 1, + "avg_discount_percent": 18.5, + "stores_with_promos": 23 + }, + { + "date": "2024-12-17", + "total_promos": 52, + "percent_off": 32, + "dollar_off": 6, + "bogo": 10, + "bundle": 2, + "set_price": 1, + "other": 1, + "avg_discount_percent": 19.2, + "stores_with_promos": 27 + } + ], + + "summary": { + "total_promo_days": 45, + "peak_promos": 67, + "peak_date": "2024-12-24", + "avg_daily_promos": 48.2, + "most_common_type": "percent_off", + "avg_discount_percent": 18.8 + } +} +``` + +**Stacked Bar Chart Usage:** + +```typescript +// Promo breakdown stacked bar +const promoTypes = ['percent_off', 'dollar_off', 'bogo', 'bundle', 'set_price', 'other']; + + + {promoTypes.map((type, i) => ( + + ))} + +``` + +**Response (Empty State - 200):** + +```json +{ + "brand_key": "cb_99999", + "period": { + "from": "2024-12-16", + "to": "2025-01-15", + "interval": "day", + "points_count": 0 + }, + "points": [], + "summary": { + "total_promo_days": 0, + "peak_promos": null, + "peak_date": null, + "avg_daily_promos": 0, + "most_common_type": null, + "avg_discount_percent": null + } +} +``` + +--- + +## 5. Store Changes Endpoint + +### `GET /v1/brands/:brand_key/dashboard/store-changes` + +Returns recent store additions and removals for "New Stores" / "Lost Stores" widgets. + +**Query Parameters:** + +| Param | Type | Required | Default | Description | +|-------|------|----------|---------|-------------| +| `days` | number | No | `30` | Lookback period (max 90) | +| `limit` | number | No | `10` | Max results per list | + +**Response (Success - 200):** + +```json +{ + "brand_key": "cb_12345", + "period": { + "days": 30, + "from": "2024-12-16", + "to": "2025-01-15" + }, + + "stores_added": { + "count": 15, + "list": [ + { + "store_id": 142, + "store_name": "Green Thumb LA", + "store_slug": "green-thumb-la", + "dispensary_name": "Green Thumb Dispensary", + "city": "Los Angeles", + "state": "CA", + "date_added": "2025-01-14", + "days_ago": 1, + "initial_sku_count": 12 + }, + { + "store_id": 156, + "store_name": "Harvest House SF", + "store_slug": "harvest-house-sf", + "dispensary_name": "Harvest House", + "city": "San Francisco", + "state": "CA", + "date_added": "2025-01-12", + "days_ago": 3, + "initial_sku_count": 8 + } + ] + }, + + "stores_removed": { + "count": 2, + "list": [ + { + "store_id": 78, + "store_name": "MedMen Venice", + "store_slug": "medmen-venice", + "dispensary_name": "MedMen", + "city": "Venice", + "state": "CA", + "date_removed": "2025-01-10", + "days_ago": 5, + "tenure_days": 156, + "last_sku_count": 6 + } + ] + }, + + "stores_reappeared": { + "count": 1, + "list": [ + { + "store_id": 92, + "store_name": "The Cannabist - Sacramento", + "store_slug": "cannabist-sacramento", + "dispensary_name": "The Cannabist", + "city": "Sacramento", + "state": "CA", + "date_reappeared": "2025-01-08", + "days_ago": 7, + "days_absent": 21, + "sku_count": 4 + } + ] + } +} +``` + +**Widget Mapping:** + +| Widget | Data Source | +|--------|-------------| +| "New Stores" badge count | `stores_added.count` | +| "New Stores" list | `stores_added.list` | +| "Lost Stores" badge count | `stores_removed.count` | +| "Lost Stores" list | `stores_removed.list` | +| "Returned" badge count | `stores_reappeared.count` | + +**Response (Empty State - 200):** + +```json +{ + "brand_key": "cb_99999", + "period": { + "days": 30, + "from": "2024-12-16", + "to": "2025-01-15" + }, + "stores_added": { + "count": 0, + "list": [] + }, + "stores_removed": { + "count": 0, + "list": [] + }, + "stores_reappeared": { + "count": 0, + "list": [] + } +} +``` + +--- + +## 6. Stores Drill-Down Endpoint + +### `GET /v1/brands/:brand_key/dashboard/stores` + +Returns all stores carrying this brand with current metrics for a detailed table view. + +**Query Parameters:** + +| Param | Type | Required | Default | Description | +|-------|------|----------|---------|-------------| +| `status` | string | No | `active` | Filter: `active`, `removed`, `all` | +| `state` | string | No | - | Filter by state code (e.g., `CA`) | +| `sort` | string | No | `sku_count` | Sort: `sku_count`, `name`, `date_added`, `tenure` | +| `order` | string | No | `desc` | Order: `asc`, `desc` | +| `limit` | number | No | `50` | Max results (max 200) | +| `offset` | number | No | `0` | Pagination offset | +| `search` | string | No | - | Search store/dispensary name | + +**Response (Success - 200):** + +```json +{ + "brand_key": "cb_12345", + "filters": { + "status": "active", + "state": null, + "search": null + }, + "pagination": { + "total": 127, + "limit": 50, + "offset": 0, + "has_more": true + }, + + "stores": [ + { + "store_id": 42, + "store_name": "Green Thumb LA", + "store_slug": "green-thumb-la", + "dispensary_name": "Green Thumb Dispensary", + "city": "Los Angeles", + "state": "CA", + "dutchie_url": "https://dutchie.com/dispensary/green-thumb-la", + + "presence": { + "status": "active", + "first_seen": "2024-06-15", + "last_seen": "2025-01-15", + "tenure_days": 214 + }, + + "sku_counts": { + "active": 24, + "in_stock": 21, + "on_promo": 3, + "total_ever": 28 + }, + + "price_stats": { + "avg_cents": 4500, + "min_cents": 2500, + "max_cents": 8500 + }, + + "activity_7d": { + "new_skus": 2, + "removed_skus": 0, + "price_changes": 1 + } + } + ], + + "aggregates": { + "total_stores": 127, + "states_count": 8, + "avg_skus_per_store": 4.3, + "top_states": [ + { "state": "CA", "count": 45 }, + { "state": "CO", "count": 28 }, + { "state": "WA", "count": 18 } + ] + } +} +``` + +**Table Column Mapping:** + +| Column | Field | +|--------|-------| +| Store Name | `store_name` (link to `dutchie_url`) | +| Location | `city`, `state` | +| SKUs | `sku_counts.active` | +| In Stock | `sku_counts.in_stock` | +| On Promo | `sku_counts.on_promo` | +| Avg Price | `price_stats.avg_cents` / 100 | +| Since | `presence.first_seen` | +| Tenure | `presence.tenure_days` + " days" | + +**Response (Empty State - 200):** + +```json +{ + "brand_key": "cb_99999", + "filters": { + "status": "active", + "state": null, + "search": null + }, + "pagination": { + "total": 0, + "limit": 50, + "offset": 0, + "has_more": false + }, + "stores": [], + "aggregates": { + "total_stores": 0, + "states_count": 0, + "avg_skus_per_store": 0, + "top_states": [] + } +} +``` + +--- + +## 7. Products Drill-Down Endpoint + +### `GET /v1/brands/:brand_key/dashboard/products` + +Returns all products for this brand with lifecycle and pricing data. + +**Query Parameters:** + +| Param | Type | Required | Default | Description | +|-------|------|----------|---------|-------------| +| `store_slug` | string | No | - | Filter by specific store | +| `status` | string | No | `active` | Filter: `active`, `stale`, `removed`, `all` | +| `in_stock` | string | No | - | Filter: `true`, `false` | +| `on_promo` | string | No | - | Filter: `true`, `false` | +| `sort` | string | No | `store_count` | Sort: `store_count`, `name`, `price`, `date_added` | +| `order` | string | No | `desc` | Order: `asc`, `desc` | +| `limit` | number | No | `50` | Max results (max 200) | +| `offset` | number | No | `0` | Pagination offset | +| `search` | string | No | - | Search product name | + +**Response (Success - 200):** + +```json +{ + "brand_key": "cb_12345", + "filters": { + "store_slug": null, + "status": "active", + "in_stock": null, + "on_promo": null, + "search": null + }, + "pagination": { + "total": 543, + "limit": 50, + "offset": 0, + "has_more": true + }, + + "products": [ + { + "product_id": 12345, + "fingerprint": "a1b2c3d4e5f6...", + "name": "Live Resin - Slurm OG", + "full_name": "Raw Garden Live Resin - Slurm OG", + "variant": "1g", + "weight": "1g", + + "distribution": { + "store_count_current": 45, + "store_count_previous_30d": 38, + "store_count_delta": 7, + "store_count_direction": "up", + "states": ["CA", "CO", "WA", "NV"] + }, + + "pricing": { + "avg_price_cents": 4500, + "min_price_cents": 3999, + "max_price_cents": 5499, + "price_spread_percent": 37.5, + "avg_formatted": "$45.00", + "min_formatted": "$39.99", + "max_formatted": "$54.99" + }, + + "stock": { + "in_stock_count": 42, + "out_of_stock_count": 3, + "in_stock_percent": 93.3 + }, + + "promos": { + "stores_on_promo": 8, + "percent_on_promo": 17.8, + "percent_time_on_special_30d": 23.5, + "common_promo_type": "percent_off" + }, + + "lifecycle": { + "first_seen": "2024-08-10", + "days_tracked": 158, + "status": "active" + } + } + ], + + "aggregates": { + "total_products": 543, + "total_in_stock": 489, + "total_on_promo": 34, + "avg_stores_per_product": 4.2, + "categories": [ + { "name": "Concentrates", "count": 312 }, + { "name": "Vapes", "count": 156 }, + { "name": "Flower", "count": 75 } + ] + } +} +``` + +**Table Column Mapping:** + +| Column | Field | +|--------|-------| +| Product | `name` | +| Size | `variant` or `weight` | +| Stores | `distribution.store_count_current` | +| Avg Price | `pricing.avg_formatted` | +| Price Range | `pricing.min_formatted` - `pricing.max_formatted` | +| In Stock % | `stock.in_stock_percent` | +| On Promo | `promos.stores_on_promo` stores | +| First Seen | `lifecycle.first_seen` | + +**Response (Empty State - 200):** + +```json +{ + "brand_key": "cb_99999", + "filters": { + "store_slug": null, + "status": "active", + "in_stock": null, + "on_promo": null, + "search": null + }, + "pagination": { + "total": 0, + "limit": 50, + "offset": 0, + "has_more": false + }, + "products": [], + "aggregates": { + "total_products": 0, + "total_in_stock": 0, + "total_on_promo": 0, + "avg_stores_per_product": 0, + "categories": [] + } +} +``` + +--- + +## 8. Empty State Behavior Summary + +All endpoints return HTTP 200 with structured empty data rather than errors when a brand has no data: + +| Endpoint | Empty Indicator | UI Treatment | +|----------|-----------------|--------------| +| `/dashboard/summary` | `store_footprint.current === 0` | Show "No data yet" card overlay | +| `/dashboard/store-timeseries` | `points.length === 0` | Show empty chart with "No data" message | +| `/dashboard/promo-timeseries` | `points.length === 0` | Show empty chart with "No data" message | +| `/dashboard/store-changes` | All `.count === 0` | Show "No changes in this period" | +| `/dashboard/stores` | `pagination.total === 0` | Show empty state illustration | +| `/dashboard/products` | `pagination.total === 0` | Show empty state illustration | + +**Empty State Detection (TypeScript):** + +```typescript +function hasBrandData(summary: DashboardSummary): boolean { + return summary.store_footprint.current > 0 || summary.active_skus.current > 0; +} + +function hasTimeseriesData(data: TimeseriesResponse): boolean { + return data.points.length > 0; +} + +function hasStoreChanges(data: StoreChangesResponse): boolean { + return data.stores_added.count > 0 || + data.stores_removed.count > 0 || + data.stores_reappeared.count > 0; +} +``` + +--- + +## 9. Error Responses + +All endpoints use consistent error response format: + +### 401 Unauthorized + +```json +{ + "error": "Unauthorized", + "code": "AUTH_REQUIRED", + "message": "Valid authentication token required" +} +``` + +### 403 Forbidden + +```json +{ + "error": "Access denied", + "code": "BRAND_ACCESS_DENIED", + "message": "You do not have access to brand: cb_12345" +} +``` + +### 404 Brand Not Found + +```json +{ + "error": "Brand not found", + "code": "BRAND_NOT_FOUND", + "message": "No brand found with key: cb_unknown" +} +``` + +### 404 Feature Disabled + +```json +{ + "error": "Feature not available", + "code": "FEATURE_DISABLED", + "message": "Brand intelligence features are temporarily unavailable" +} +``` + +### 422 Validation Error + +```json +{ + "error": "Validation error", + "code": "INVALID_PARAMS", + "message": "Invalid parameters", + "details": { + "window": "Must be one of: 7d, 30d, 90d", + "limit": "Must be between 1 and 200" + } +} +``` + +### 500 Server Error + +```json +{ + "error": "Internal server error", + "code": "INTERNAL_ERROR", + "message": "An unexpected error occurred", + "request_id": "req_abc123" +} +``` + +--- + +## 10. TypeScript Response Types + +```typescript +// Common types +type Direction = 'up' | 'down' | 'stable'; + +interface DeltaMetric { + current: number; + previous: number; + delta: number; + delta_percent: number; + direction: Direction; +} + +interface NullableDeltaMetric { + current: number | null; + previous: number | null; + delta: number | null; + direction: Direction; +} + +// Summary endpoint +interface DashboardSummary { + brand_key: string; + brand_name: string; + as_of: string; + window: '7d' | '30d' | '90d'; + store_footprint: DeltaMetric; + active_skus: DeltaMetric; + in_stock_rate: NullableDeltaMetric; + promo_exposure: { + skus_on_promo: number; + stores_running_promos: number; + percent_of_skus: number; + direction: Direction; + }; + price_summary: { + avg_cents: number | null; + min_cents: number | null; + max_cents: number | null; + avg_formatted: string | null; + min_formatted: string | null; + max_formatted: string | null; + }; + activity_last_7d: { + stores_added: number; + stores_removed: number; + new_skus: number; + removed_skus: number; + price_changes: number; + }; +} + +// Timeseries endpoints +interface TimeseriesPeriod { + from: string; + to: string; + interval: 'day' | 'week' | 'month'; + points_count: number; +} + +interface StoreTimeseriesPoint { + date: string; + stores: number; + stores_added: number; + stores_removed: number; + stores_net: number; + skus: number; + skus_in_stock: number; + skus_new: number; + skus_removed: number; + skus_net: number; +} + +interface StoreTimeseriesResponse { + brand_key: string; + period: TimeseriesPeriod; + points: StoreTimeseriesPoint[]; + summary: { + stores_start: number | null; + stores_end: number | null; + stores_peak: number | null; + stores_low: number | null; + skus_start: number | null; + skus_end: number | null; + skus_peak: number | null; + skus_low: number | null; + }; +} + +interface PromoTimeseriesPoint { + date: string; + total_promos: number; + percent_off: number; + dollar_off: number; + bogo: number; + bundle: number; + set_price: number; + other: number; + avg_discount_percent: number; + stores_with_promos: number; +} + +interface PromoTimeseriesResponse { + brand_key: string; + period: TimeseriesPeriod; + points: PromoTimeseriesPoint[]; + summary: { + total_promo_days: number; + peak_promos: number | null; + peak_date: string | null; + avg_daily_promos: number; + most_common_type: string | null; + avg_discount_percent: number | null; + }; +} + +// Store changes +interface StoreChangeEntry { + store_id: number; + store_name: string; + store_slug: string; + dispensary_name: string; + city: string; + state: string; +} + +interface StoreAddedEntry extends StoreChangeEntry { + date_added: string; + days_ago: number; + initial_sku_count: number; +} + +interface StoreRemovedEntry extends StoreChangeEntry { + date_removed: string; + days_ago: number; + tenure_days: number; + last_sku_count: number; +} + +interface StoreReappearedEntry extends StoreChangeEntry { + date_reappeared: string; + days_ago: number; + days_absent: number; + sku_count: number; +} + +interface StoreChangesResponse { + brand_key: string; + period: { + days: number; + from: string; + to: string; + }; + stores_added: { + count: number; + list: StoreAddedEntry[]; + }; + stores_removed: { + count: number; + list: StoreRemovedEntry[]; + }; + stores_reappeared: { + count: number; + list: StoreReappearedEntry[]; + }; +} + +// Stores drill-down +interface StoreListEntry { + store_id: number; + store_name: string; + store_slug: string; + dispensary_name: string; + city: string; + state: string; + dutchie_url: string; + presence: { + status: 'active' | 'removed'; + first_seen: string; + last_seen: string; + tenure_days: number; + }; + sku_counts: { + active: number; + in_stock: number; + on_promo: number; + total_ever: number; + }; + price_stats: { + avg_cents: number; + min_cents: number; + max_cents: number; + }; + activity_7d: { + new_skus: number; + removed_skus: number; + price_changes: number; + }; +} + +interface StoresResponse { + brand_key: string; + filters: { + status: string; + state: string | null; + search: string | null; + }; + pagination: { + total: number; + limit: number; + offset: number; + has_more: boolean; + }; + stores: StoreListEntry[]; + aggregates: { + total_stores: number; + states_count: number; + avg_skus_per_store: number; + top_states: { state: string; count: number }[]; + }; +} + +// Products drill-down +interface ProductListEntry { + product_id: number; + fingerprint: string; + name: string; + full_name: string; + variant: string; + weight: string; + distribution: { + store_count_current: number; + store_count_previous_30d: number; + store_count_delta: number; + store_count_direction: Direction; + states: string[]; + }; + pricing: { + avg_price_cents: number; + min_price_cents: number; + max_price_cents: number; + price_spread_percent: number; + avg_formatted: string; + min_formatted: string; + max_formatted: string; + }; + stock: { + in_stock_count: number; + out_of_stock_count: number; + in_stock_percent: number; + }; + promos: { + stores_on_promo: number; + percent_on_promo: number; + percent_time_on_special_30d: number; + common_promo_type: string | null; + }; + lifecycle: { + first_seen: string; + days_tracked: number; + status: 'active' | 'stale' | 'removed'; + }; +} + +interface ProductsResponse { + brand_key: string; + filters: { + store_slug: string | null; + status: string; + in_stock: string | null; + on_promo: string | null; + search: string | null; + }; + pagination: { + total: number; + limit: number; + offset: number; + has_more: boolean; + }; + products: ProductListEntry[]; + aggregates: { + total_products: number; + total_in_stock: number; + total_on_promo: number; + avg_stores_per_product: number; + categories: { name: string; count: number }[]; + }; +} +``` + +--- + +## Document Version + +| Version | Date | Author | Changes | +|---------|------|--------|---------| +| 1.0 | 2025-01-15 | Claude | Initial front-end ready specification | diff --git a/docs/CRAWL_OPERATIONS.md b/docs/CRAWL_OPERATIONS.md new file mode 100644 index 00000000..0af3cccc --- /dev/null +++ b/docs/CRAWL_OPERATIONS.md @@ -0,0 +1,592 @@ +# Crawl Operations & Data Philosophy + +This document defines the operational constraints, scheduling requirements, and data integrity philosophy for the dispensary scraper system. + +--- + +## 1. Frozen Crawler Policy + +> **CRITICAL CONSTRAINT**: The crawler code is FROZEN. Do NOT modify any crawler logic. + +### What Is Frozen + +The following components are read-only and must not be modified: + +- **Selectors**: All CSS/XPath selectors for extracting data from Dutchie pages +- **Parsing Logic**: Functions that transform raw HTML into structured data +- **Request Patterns**: URL construction, pagination, API calls to Dutchie +- **Browser Configuration**: Puppeteer settings, user agents, viewport sizes +- **Rate Limiting**: Request delays, retry logic, concurrent request limits + +### What CAN Be Modified + +You may build around the crawler's output: + +| Layer | Allowed Changes | +|-------|-----------------| +| **Scheduling** | CronJobs, run frequency, store queuing | +| **Ingestion** | Post-processing of crawler output before DB insert | +| **API Layer** | Query logic, computed fields, response transformations | +| **Intelligence** | Aggregation tables, metrics computation | +| **Infrastructure** | K8s resources, scaling, monitoring | + +### Rationale + +The crawler has been stabilized through extensive testing. Changes to selectors or parsing risk: +- Breaking data extraction if Dutchie changes their UI +- Introducing regressions that are hard to detect +- Requiring re-validation across all store types + +All improvements must happen in **downstream processing**, not in the crawler itself. + +--- + +## 2. Crawl Scheduling + +### Standard Schedule: Every 4 Hours + +Run a full crawl for each store every 4 hours, 24/7. + +```yaml +# K8s CronJob: Every 4 hours +apiVersion: batch/v1 +kind: CronJob +metadata: + name: scraper-4h-cycle + namespace: dispensary-scraper +spec: + schedule: "0 */4 * * *" # 00:00, 04:00, 08:00, 12:00, 16:00, 20:00 UTC + concurrencyPolicy: Forbid + jobTemplate: + spec: + template: + spec: + containers: + - name: scraper + image: code.cannabrands.app/creationshop/dispensary-scraper:latest + command: ["node", "dist/scripts/run-all-stores.js"] + env: + - name: DATABASE_URL + valueFrom: + secretKeyRef: + name: scraper-secrets + key: database-url + restartPolicy: OnFailure +``` + +### Daily Specials Crawl: 12:01 AM Store Local Time + +Dispensaries often update their daily specials at midnight. We ensure a crawl happens at 12:01 AM in each store's local timezone. + +```yaml +# K8s CronJob: Daily specials at store midnight (example for MST/Arizona) +apiVersion: batch/v1 +kind: CronJob +metadata: + name: scraper-daily-specials-mst + namespace: dispensary-scraper +spec: + schedule: "1 7 * * *" # 12:01 AM MST = 07:01 UTC + concurrencyPolicy: Forbid + jobTemplate: + spec: + template: + spec: + containers: + - name: scraper + image: code.cannabrands.app/creationshop/dispensary-scraper:latest + command: ["node", "dist/scripts/run-stores-by-timezone.js", "America/Phoenix"] + restartPolicy: OnFailure +``` + +### Timezone-Aware Scheduling + +Stores table includes timezone information: + +```sql +ALTER TABLE stores ADD COLUMN IF NOT EXISTS timezone VARCHAR(50) DEFAULT 'America/Phoenix'; + +-- Lookup table for common dispensary timezones +-- America/Phoenix (Arizona, no DST) +-- America/Los_Angeles (California) +-- America/Denver (Colorado) +-- America/Chicago (Illinois) +``` + +### Scripts Required + +``` +/backend/src/scripts/ +├── run-all-stores.ts # Run crawl for all enabled stores +├── run-stores-by-timezone.ts # Run crawl for stores in a specific timezone +└── scheduler.ts # Orchestrates CronJob dispatch +``` + +--- + +## 3. Specials Detection Logic + +> **Problem**: The Specials tab in the frontend is EMPTY even though products have discounts. + +### Root Cause Analysis + +Database investigation reveals: + +| Metric | Count | +|--------|-------| +| Total products | 1,414 | +| `is_special = true` | 0 | +| Has "Special Offer" in name | 325 | +| Has `sale_price < regular_price` | 4 | + +The crawler captures "Special Offer" **embedded in the product name** but doesn't set `is_special = true`. + +### Solution: API-Layer Specials Detection + +Since the crawler is frozen, detect specials at query time: + +```sql +-- Computed is_on_special in API queries +SELECT + p.*, + CASE + WHEN p.name ILIKE '%Special Offer%' THEN TRUE + WHEN p.sale_price IS NOT NULL + AND p.regular_price IS NOT NULL + AND p.sale_price::numeric < p.regular_price::numeric THEN TRUE + WHEN p.price IS NOT NULL + AND p.original_price IS NOT NULL + AND p.price::numeric < p.original_price::numeric THEN TRUE + ELSE FALSE + END AS is_on_special, + + -- Compute special type + CASE + WHEN p.name ILIKE '%Special Offer%' THEN 'special_offer' + WHEN p.sale_price IS NOT NULL + AND p.regular_price IS NOT NULL + AND p.sale_price::numeric < p.regular_price::numeric THEN 'percent_off' + ELSE NULL + END AS computed_special_type, + + -- Compute discount percentage + CASE + WHEN p.sale_price IS NOT NULL + AND p.regular_price IS NOT NULL + AND p.regular_price::numeric > 0 + THEN ROUND((1 - p.sale_price::numeric / p.regular_price::numeric) * 100, 0) + ELSE NULL + END AS computed_discount_percent + +FROM products p +WHERE p.store_id = :store_id; +``` + +### Special Detection Rules (Priority Order) + +1. **Name Contains "Special Offer"**: `name ILIKE '%Special Offer%'` + - Type: `special_offer` + - Badge: "Special" + +2. **Price Discount (sale < regular)**: `sale_price < regular_price` + - Type: `percent_off` + - Badge: Computed as "X% OFF" + +3. **Price Discount (current < original)**: `price < original_price` + - Type: `percent_off` + - Badge: Computed as "X% OFF" + +4. **Metadata Offers** (future): `metadata->'offers' IS NOT NULL` + - Parse offer type from metadata JSON + +### Clean Product Name + +Strip "Special Offer" from display name: + +```typescript +function cleanProductName(rawName: string): string { + return rawName + .replace(/Special Offer$/i, '') + .replace(/\s+$/, '') // Trim trailing whitespace + .trim(); +} +``` + +### API Specials Endpoint + +```typescript +// GET /api/stores/:store_key/specials +async function getStoreSpecials(storeKey: string, options: SpecialsOptions) { + const query = ` + WITH specials AS ( + SELECT + p.*, + -- Detect special + CASE + WHEN p.name ILIKE '%Special Offer%' THEN TRUE + WHEN p.sale_price::numeric < p.regular_price::numeric THEN TRUE + ELSE FALSE + END AS is_on_special, + + -- Compute discount + CASE + WHEN p.sale_price IS NOT NULL AND p.regular_price IS NOT NULL + THEN ROUND((1 - p.sale_price::numeric / p.regular_price::numeric) * 100) + ELSE NULL + END AS discount_percent + + FROM products p + JOIN stores s ON p.store_id = s.id + WHERE s.store_key = $1 + AND p.in_stock = TRUE + ) + SELECT * FROM specials + WHERE is_on_special = TRUE + ORDER BY discount_percent DESC NULLS LAST + LIMIT $2 OFFSET $3 + `; + + return db.query(query, [storeKey, options.limit, options.offset]); +} +``` + +--- + +## 4. Append-Only Data Philosophy + +> **Principle**: Every crawl should ADD information, never LOSE it. + +### What Append-Only Means + +| Action | Allowed | Not Allowed | +|--------|---------|-------------| +| Insert new product | ✅ | - | +| Update product price | ✅ | - | +| Mark product out-of-stock | ✅ | - | +| DELETE product row | ❌ | Never delete | +| TRUNCATE table | ❌ | Never truncate | +| UPDATE to remove data | ❌ | Never null-out existing data | + +### Product Lifecycle States + +```sql +-- Products are never deleted, only state changes +ALTER TABLE products ADD COLUMN IF NOT EXISTS status VARCHAR(20) DEFAULT 'active'; + +-- Statuses: +-- 'active' - Currently in stock or recently seen +-- 'out_of_stock' - Seen but marked out of stock +-- 'stale' - Not seen in last 3 crawls (likely discontinued) +-- 'archived' - Manually marked as discontinued + +CREATE INDEX idx_products_status ON products(status); +``` + +### Marking Products Stale (NOT Deleting) + +```typescript +// After crawl completes, mark unseen products as stale +async function markStaleProducts(storeId: number, crawlRunId: number) { + await db.query(` + UPDATE products + SET + status = 'stale', + updated_at = NOW() + WHERE store_id = $1 + AND id NOT IN ( + SELECT DISTINCT product_id + FROM store_product_snapshots + WHERE crawl_run_id = $2 + ) + AND status = 'active' + AND last_seen_at < NOW() - INTERVAL '3 days' + `, [storeId, crawlRunId]); +} +``` + +### Store Product Snapshots: True Append-Only + +The `store_product_snapshots` table is strictly append-only: + +```sql +CREATE TABLE store_product_snapshots ( + id SERIAL PRIMARY KEY, + store_id INTEGER NOT NULL REFERENCES stores(id), + product_id INTEGER NOT NULL REFERENCES products(id), + crawl_run_id INTEGER NOT NULL REFERENCES crawl_runs(id), + + -- Snapshot of data at crawl time + price_cents INTEGER, + regular_price_cents INTEGER, + sale_price_cents INTEGER, + in_stock BOOLEAN NOT NULL, + + -- Computed at crawl time + is_on_special BOOLEAN NOT NULL DEFAULT FALSE, + special_type VARCHAR(50), + discount_percent INTEGER, + + captured_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + + -- Composite unique: one snapshot per product per crawl + CONSTRAINT uq_snapshot_product_crawl UNIQUE (product_id, crawl_run_id) +); + +-- NO UPDATE or DELETE triggers - this table is INSERT-only +-- For data corrections, insert a new snapshot with corrected flag + +CREATE INDEX idx_snapshots_crawl ON store_product_snapshots(crawl_run_id); +CREATE INDEX idx_snapshots_product_time ON store_product_snapshots(product_id, captured_at DESC); +``` + +### Crawl Runs Table + +Track every crawl execution: + +```sql +CREATE TABLE crawl_runs ( + id SERIAL PRIMARY KEY, + store_id INTEGER NOT NULL REFERENCES stores(id), + started_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + completed_at TIMESTAMPTZ, + status VARCHAR(20) NOT NULL DEFAULT 'running', + products_found INTEGER, + products_new INTEGER, + products_updated INTEGER, + error_message TEXT, + + -- Scheduling metadata + trigger_type VARCHAR(20) NOT NULL DEFAULT 'scheduled', -- 'scheduled', 'manual', 'daily_specials' + + CONSTRAINT chk_crawl_status CHECK (status IN ('running', 'completed', 'failed')) +); + +CREATE INDEX idx_crawl_runs_store_time ON crawl_runs(store_id, started_at DESC); +``` + +### Data Correction Pattern + +If data needs correction, don't UPDATE - insert a correction record: + +```sql +CREATE TABLE data_corrections ( + id SERIAL PRIMARY KEY, + table_name VARCHAR(50) NOT NULL, + record_id INTEGER NOT NULL, + field_name VARCHAR(100) NOT NULL, + old_value JSONB, + new_value JSONB, + reason TEXT NOT NULL, + corrected_by VARCHAR(100) NOT NULL, + corrected_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); +``` + +--- + +## 5. Safe Ingestion Patterns + +### Upsert Products (Preserving History) + +```typescript +async function upsertProduct(storeId: number, crawlRunId: number, product: ScrapedProduct) { + // 1. Find or create product + const existing = await db.query( + `SELECT id, price, regular_price, sale_price FROM products + WHERE store_id = $1 AND dutchie_product_id = $2`, + [storeId, product.dutchieId] + ); + + let productId: number; + + if (existing.rows.length === 0) { + // INSERT new product + const result = await db.query(` + INSERT INTO products ( + store_id, dutchie_product_id, name, slug, price, regular_price, sale_price, + in_stock, first_seen_at, last_seen_at, status + ) VALUES ($1, $2, $3, $4, $5, $6, $7, TRUE, NOW(), NOW(), 'active') + RETURNING id + `, [storeId, product.dutchieId, product.name, product.slug, + product.price, product.regularPrice, product.salePrice]); + productId = result.rows[0].id; + } else { + // UPDATE existing - only update if values changed, never null-out + productId = existing.rows[0].id; + await db.query(` + UPDATE products SET + name = COALESCE($2, name), + price = COALESCE($3, price), + regular_price = COALESCE($4, regular_price), + sale_price = COALESCE($5, sale_price), + in_stock = TRUE, + last_seen_at = NOW(), + status = 'active', + updated_at = NOW() + WHERE id = $1 + `, [productId, product.name, product.price, product.regularPrice, product.salePrice]); + } + + // 2. Always create snapshot (append-only) + const isOnSpecial = detectSpecial(product); + const discountPercent = computeDiscount(product); + + await db.query(` + INSERT INTO store_product_snapshots ( + store_id, product_id, crawl_run_id, + price_cents, regular_price_cents, sale_price_cents, + in_stock, is_on_special, special_type, discount_percent + ) VALUES ($1, $2, $3, $4, $5, $6, TRUE, $7, $8, $9) + ON CONFLICT (product_id, crawl_run_id) DO NOTHING + `, [ + storeId, productId, crawlRunId, + toCents(product.price), toCents(product.regularPrice), toCents(product.salePrice), + isOnSpecial, isOnSpecial ? 'percent_off' : null, discountPercent + ]); + + return productId; +} + +function detectSpecial(product: ScrapedProduct): boolean { + // Check name for "Special Offer" + if (product.name?.includes('Special Offer')) return true; + + // Check price discount + if (product.salePrice && product.regularPrice) { + return parseFloat(product.salePrice) < parseFloat(product.regularPrice); + } + + return false; +} + +function computeDiscount(product: ScrapedProduct): number | null { + if (!product.salePrice || !product.regularPrice) return null; + + const sale = parseFloat(product.salePrice); + const regular = parseFloat(product.regularPrice); + + if (regular <= 0) return null; + + return Math.round((1 - sale / regular) * 100); +} +``` + +--- + +## 6. K8s Deployment Configuration + +### CronJobs Overview + +```yaml +# All CronJobs for scheduling +apiVersion: v1 +kind: List +items: + # 1. Standard 4-hour crawl cycle + - apiVersion: batch/v1 + kind: CronJob + metadata: + name: scraper-4h-00 + namespace: dispensary-scraper + spec: + schedule: "0 0,4,8,12,16,20 * * *" + concurrencyPolicy: Forbid + successfulJobsHistoryLimit: 3 + failedJobsHistoryLimit: 3 + jobTemplate: + spec: + activeDeadlineSeconds: 3600 # 1 hour timeout + template: + spec: + containers: + - name: scraper + image: code.cannabrands.app/creationshop/dispensary-scraper:latest + command: ["node", "dist/scripts/run-all-stores.js"] + resources: + requests: + memory: "512Mi" + cpu: "250m" + limits: + memory: "2Gi" + cpu: "1000m" + restartPolicy: OnFailure + + # 2. Daily specials crawl - Arizona (MST, no DST) + - apiVersion: batch/v1 + kind: CronJob + metadata: + name: scraper-daily-mst + namespace: dispensary-scraper + spec: + schedule: "1 7 * * *" # 12:01 AM MST = 07:01 UTC + concurrencyPolicy: Forbid + jobTemplate: + spec: + template: + spec: + containers: + - name: scraper + command: ["node", "dist/scripts/run-stores-by-timezone.js", "America/Phoenix"] + + # 3. Daily specials crawl - California (PST/PDT) + - apiVersion: batch/v1 + kind: CronJob + metadata: + name: scraper-daily-pst + namespace: dispensary-scraper + spec: + schedule: "1 8 * * *" # 12:01 AM PST = 08:01 UTC (adjust for DST) + concurrencyPolicy: Forbid + jobTemplate: + spec: + template: + spec: + containers: + - name: scraper + command: ["node", "dist/scripts/run-stores-by-timezone.js", "America/Los_Angeles"] +``` + +### Monitoring and Alerts + +```yaml +# PrometheusRule for scraper monitoring +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: scraper-alerts + namespace: dispensary-scraper +spec: + groups: + - name: scraper.rules + rules: + - alert: ScraperJobFailed + expr: kube_job_status_failed{namespace="dispensary-scraper"} > 0 + for: 5m + labels: + severity: warning + annotations: + summary: "Scraper job failed" + + - alert: ScraperMissedSchedule + expr: time() - kube_cronjob_status_last_successful_time{namespace="dispensary-scraper"} > 18000 + for: 10m + labels: + severity: critical + annotations: + summary: "Scraper hasn't run successfully in 5+ hours" +``` + +--- + +## 7. Summary + +| Constraint | Implementation | +|------------|----------------| +| **Frozen Crawler** | No changes to selectors, parsing, or request logic | +| **4-Hour Schedule** | K8s CronJob at 0,4,8,12,16,20 UTC | +| **12:01 AM Specials** | Timezone-specific CronJobs for store local midnight | +| **Specials Detection** | API-layer detection via name pattern + price comparison | +| **Append-Only Data** | Never DELETE; use status flags; `store_product_snapshots` is INSERT-only | +| **Historical Preservation** | All crawls create snapshots; stale products marked, never deleted | + +This design ensures we maximize the value of crawler data without risking breakage from crawler modifications. diff --git a/docs/PRODUCT_BRAND_INTELLIGENCE_ARCHITECTURE.md b/docs/PRODUCT_BRAND_INTELLIGENCE_ARCHITECTURE.md new file mode 100644 index 00000000..5a23e050 --- /dev/null +++ b/docs/PRODUCT_BRAND_INTELLIGENCE_ARCHITECTURE.md @@ -0,0 +1,2004 @@ +# Product & Brand Intelligence Layer - Consolidated Architecture + +## Overview + +This document describes an **additive intelligence layer** that tracks product and brand lifecycle across dispensary stores, powers analytics dashboards, and exposes brand-scoped APIs for the Cannabrands B2B SaaS platform. + +### Non-Negotiable Constraints + +1. **DO NOT** modify or redesign the crawler +2. **DO NOT** break existing API endpoints +3. **DO NOT** break the WordPress plugin integration +4. Intelligence layer must be **additive only** +5. Rollback must be possible with **feature flags**, not schema deletion + +### Key Architecture Decisions + +| Decision | Rationale | +|----------|-----------| +| Crawler unchanged | Existing scraping logic is stable; intelligence is post-processing | +| WP plugin unchanged | Store-facing API remains identical | +| Store-facing API unchanged | No breaking changes to existing consumers | +| Additive schema | New tables only; no modifications to existing tables | +| Brand-scoped queries | Multi-tenant isolation for Cannabrands platform | +| Feature flag rollback | Zero-code-change disable capability | + +### Feature Flags (Config) + +| Flag | Purpose | +|------|---------| +| `INTELLIGENCE_ENABLED` | Master kill switch | +| `INTELLIGENCE_INGEST_ENABLED` | Controls ingestion into intelligence tables | +| `INTELLIGENCE_API_ENABLED` | Controls brand intelligence API endpoints | +| `BRAND_METRICS_ENABLED` | Controls daily aggregation job | + +**Normal rollback = flip flags, not drop tables.** + +--- + +## 1. Data Model + +### 1.1 New ENUMs + +```sql +-- Product change classification +CREATE TYPE product_change_type AS ENUM ( + 'NEW', -- First time seen at this store + 'CHANGED', -- Price, stock, or special changed + 'UNCHANGED', -- No changes detected + 'STALE', -- Missing from recent crawls (3+ consecutive) + 'REMOVED' -- Missing for extended period (7+ consecutive) +); + +-- Special/deal types +CREATE TYPE special_type AS ENUM ( + 'percent_off', + 'dollar_off', + 'bogo', + 'bundle', + 'set_price', + 'other' +); + +-- Brand event types +CREATE TYPE brand_event_type AS ENUM ( + 'ADDED', -- Brand appeared at store + 'REMOVED', -- Brand disappeared from store + 'REAPPEARED' -- Brand returned after removal +); +``` + +### 1.2 Crawl Tracking + +```sql +CREATE TABLE crawl_runs ( + id SERIAL PRIMARY KEY, + store_id INTEGER NOT NULL REFERENCES stores(id), + dispensary_id INTEGER NOT NULL REFERENCES dispensaries(id), + started_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + completed_at TIMESTAMPTZ, + status VARCHAR(20) NOT NULL DEFAULT 'running', -- running, completed, failed + products_found INTEGER, + products_new INTEGER, + products_changed INTEGER, + products_stale INTEGER, + error_message TEXT, + metadata JSONB DEFAULT '{}' +); + +CREATE INDEX idx_crawl_runs_store ON crawl_runs(store_id, started_at DESC); +CREATE INDEX idx_crawl_runs_dispensary ON crawl_runs(dispensary_id, started_at DESC); +``` + +### 1.3 Canonical Brands + +```sql +CREATE TABLE canonical_brands ( + id SERIAL PRIMARY KEY, + name VARCHAR(255) NOT NULL UNIQUE, + slug VARCHAR(255) NOT NULL UNIQUE, + aliases TEXT[] DEFAULT '{}', + + -- Cannabrands integration fields + cannabrands_id VARCHAR(100) UNIQUE, -- Stable primary mapping to Cannabrands internal brand ID + cannabrands_slug VARCHAR(255), -- Human-readable key for friendly URLs + is_verified BOOLEAN DEFAULT FALSE, -- Admin-verified brand mapping + + metadata JSONB DEFAULT '{}', + created_at TIMESTAMPTZ DEFAULT NOW(), + updated_at TIMESTAMPTZ DEFAULT NOW() +); + +-- Index for lookups by cannabrands_id (primary API lookup) +CREATE INDEX idx_canonical_brands_cannabrands_id ON canonical_brands(cannabrands_id) + WHERE cannabrands_id IS NOT NULL; + +-- Index for lookups by cannabrands_slug (friendly URL lookup) +CREATE INDEX idx_canonical_brands_cannabrands_slug ON canonical_brands(cannabrands_slug) + WHERE cannabrands_slug IS NOT NULL; + +CREATE INDEX idx_canonical_brands_aliases ON canonical_brands USING GIN(aliases); +``` + +**Cannabrands Mapping Notes:** +- `cannabrands_id`: The stable primary mapping to the Cannabrands app's internal brand IDs. This is the authoritative identifier used for API calls and data correlation. +- `cannabrands_slug`: A convenience field for friendly URLs and human-readable keys (e.g., `raw-garden`, `stiiizy`). +- The admin mapping UI should populate `cannabrands_id` and optionally `cannabrands_slug`. +- Only brands with a non-null `cannabrands_id` are considered "active" in the Cannabrands platform. + +### 1.4 Store-Level Product Tracking + +```sql +CREATE TABLE store_products ( + id SERIAL PRIMARY KEY, + store_id INTEGER NOT NULL REFERENCES stores(id), + dispensary_id INTEGER NOT NULL REFERENCES dispensaries(id), + product_id INTEGER NOT NULL REFERENCES products(id), + canonical_brand_id INTEGER REFERENCES canonical_brands(id), + + -- Fingerprint for deduplication + fingerprint VARCHAR(64) NOT NULL, + + -- Lifecycle tracking + first_seen_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + last_seen_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + last_changed_at TIMESTAMPTZ, + removed_at TIMESTAMPTZ, + + -- Current state + current_price NUMERIC(10,2), + current_in_stock BOOLEAN DEFAULT TRUE, + current_special_text TEXT, + current_special_type special_type, + current_special_value NUMERIC(10,2), + + -- Staleness tracking + consecutive_missing INTEGER DEFAULT 0, + change_status product_change_type DEFAULT 'NEW', + + -- Metadata + metadata JSONB DEFAULT '{}', + + UNIQUE(store_id, fingerprint) +); + +CREATE INDEX idx_store_products_store ON store_products(store_id); +CREATE INDEX idx_store_products_dispensary ON store_products(dispensary_id); +CREATE INDEX idx_store_products_brand ON store_products(canonical_brand_id); +CREATE INDEX idx_store_products_status ON store_products(change_status); +CREATE INDEX idx_store_products_fingerprint ON store_products(fingerprint); +CREATE INDEX idx_store_products_active ON store_products(store_id, change_status) + WHERE change_status NOT IN ('STALE', 'REMOVED'); +``` + +### 1.5 Product Snapshots (Historical) + +```sql +CREATE TABLE store_product_snapshots ( + id SERIAL PRIMARY KEY, + store_product_id INTEGER NOT NULL REFERENCES store_products(id) ON DELETE CASCADE, + crawl_run_id INTEGER NOT NULL REFERENCES crawl_runs(id), + + -- Point-in-time state + price NUMERIC(10,2), + in_stock BOOLEAN, + special_text TEXT, + special_type special_type, + special_value NUMERIC(10,2), + + -- Change detection + change_type product_change_type NOT NULL, + changes_detected JSONB, -- {"price": {"old": 25, "new": 30}, "in_stock": {...}} + + captured_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +CREATE INDEX idx_snapshots_store_product ON store_product_snapshots(store_product_id, captured_at DESC); +CREATE INDEX idx_snapshots_crawl ON store_product_snapshots(crawl_run_id); +CREATE INDEX idx_snapshots_change_type ON store_product_snapshots(change_type); + +-- Partition by month for performance (optional) +-- CREATE TABLE store_product_snapshots_2025_01 PARTITION OF store_product_snapshots +-- FOR VALUES FROM ('2025-01-01') TO ('2025-02-01'); +``` + +### 1.6 Brand Store Presence + +```sql +CREATE TABLE brand_store_presence ( + id SERIAL PRIMARY KEY, + canonical_brand_id INTEGER NOT NULL REFERENCES canonical_brands(id), + store_id INTEGER NOT NULL REFERENCES stores(id), + dispensary_id INTEGER NOT NULL REFERENCES dispensaries(id), + + -- Presence tracking + first_seen_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + last_seen_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + removed_at TIMESTAMPTZ, + + -- Current metrics + active_sku_count INTEGER DEFAULT 0, + in_stock_sku_count INTEGER DEFAULT 0, + total_sku_count INTEGER DEFAULT 0, -- Including removed + + -- Status + is_active BOOLEAN DEFAULT TRUE, + consecutive_missing INTEGER DEFAULT 0, + + UNIQUE(canonical_brand_id, store_id) +); + +CREATE INDEX idx_brand_presence_brand ON brand_store_presence(canonical_brand_id); +CREATE INDEX idx_brand_presence_store ON brand_store_presence(store_id); +CREATE INDEX idx_brand_presence_active ON brand_store_presence(canonical_brand_id, is_active) + WHERE is_active = TRUE; +``` + +### 1.7 Pre-Aggregated Brand Metrics (for Cannabrands) + +```sql +-- Daily brand metrics (aggregated nightly) +CREATE TABLE brand_daily_metrics ( + id SERIAL PRIMARY KEY, + canonical_brand_id INTEGER NOT NULL REFERENCES canonical_brands(id), + date DATE NOT NULL, + + -- Store counts + total_stores INTEGER DEFAULT 0, + stores_added INTEGER DEFAULT 0, + stores_removed INTEGER DEFAULT 0, + + -- SKU counts + total_skus INTEGER DEFAULT 0, + skus_in_stock INTEGER DEFAULT 0, + new_skus INTEGER DEFAULT 0, + removed_skus INTEGER DEFAULT 0, + + -- Price metrics + avg_price NUMERIC(10,2), + min_price NUMERIC(10,2), + max_price NUMERIC(10,2), + price_changes INTEGER DEFAULT 0, + + -- Promo metrics + skus_on_promo INTEGER DEFAULT 0, + promo_value_sum NUMERIC(12,2) DEFAULT 0, + + computed_at TIMESTAMPTZ DEFAULT NOW(), + + UNIQUE(canonical_brand_id, date) +); + +CREATE INDEX idx_brand_metrics_brand_date ON brand_daily_metrics(canonical_brand_id, date DESC); + +-- Brand store events (for timeline) +CREATE TABLE brand_store_events ( + id SERIAL PRIMARY KEY, + canonical_brand_id INTEGER NOT NULL REFERENCES canonical_brands(id), + store_id INTEGER NOT NULL REFERENCES stores(id), + dispensary_id INTEGER NOT NULL REFERENCES dispensaries(id), + event_type brand_event_type NOT NULL, + event_date DATE NOT NULL, + crawl_run_id INTEGER REFERENCES crawl_runs(id), -- Optional link to triggering crawl + sku_count INTEGER, + notes TEXT, + metadata JSONB DEFAULT '{}', + created_at TIMESTAMPTZ DEFAULT NOW() +); + +CREATE INDEX idx_brand_events_brand ON brand_store_events(canonical_brand_id, event_date DESC); +CREATE INDEX idx_brand_events_store ON brand_store_events(store_id, event_date DESC); +CREATE INDEX idx_brand_events_crawl ON brand_store_events(crawl_run_id) WHERE crawl_run_id IS NOT NULL; + +-- Daily promo metrics by brand +CREATE TABLE brand_promo_daily_metrics ( + id SERIAL PRIMARY KEY, + canonical_brand_id INTEGER NOT NULL REFERENCES canonical_brands(id), + date DATE NOT NULL, + + -- Promo breakdown + total_promos INTEGER DEFAULT 0, + percent_off_count INTEGER DEFAULT 0, + bogo_count INTEGER DEFAULT 0, + bundle_count INTEGER DEFAULT 0, + other_count INTEGER DEFAULT 0, + + -- Value metrics + avg_discount_pct NUMERIC(5,2), + total_discount_value NUMERIC(12,2), + + -- Store breakdown + stores_with_promos INTEGER DEFAULT 0, + + computed_at TIMESTAMPTZ DEFAULT NOW(), + + UNIQUE(canonical_brand_id, date) +); + +CREATE INDEX idx_brand_promo_metrics ON brand_promo_daily_metrics(canonical_brand_id, date DESC); +``` + +### 1.8 Brand Store Detail View + +This view provides a quick per-store drill down for a brand with pre-aggregated SKU counts and presence fields. + +```sql +CREATE OR REPLACE VIEW v_brand_store_detail AS +SELECT + cb.id AS brand_id, + cb.name AS brand_name, + cb.cannabrands_id AS brand_key, + cb.cannabrands_slug AS brand_slug, + s.id AS store_id, + s.name AS store_name, + s.slug AS store_slug, + d.id AS dispensary_id, + d.name AS dispensary_name, + d.city AS city, + d.state AS state, + d.dutchie_url AS dutchie_url, + bsp.first_seen_at AS first_seen_at, + bsp.last_seen_at AS last_seen_at, + bsp.removed_at AS removed_at, + bsp.is_active AS is_active, + bsp.active_sku_count AS active_sku_count, + bsp.in_stock_sku_count AS in_stock_sku_count, + bsp.total_sku_count AS total_sku_count, + -- Calculate days since last seen + EXTRACT(DAY FROM NOW() - bsp.last_seen_at)::INTEGER AS days_since_seen, + -- Calculate tenure in days + EXTRACT(DAY FROM COALESCE(bsp.removed_at, NOW()) - bsp.first_seen_at)::INTEGER AS tenure_days +FROM brand_store_presence bsp +JOIN canonical_brands cb ON cb.id = bsp.canonical_brand_id +JOIN stores s ON s.id = bsp.store_id +JOIN dispensaries d ON d.id = bsp.dispensary_id; + +-- Index hint: queries on this view should filter by brand_id or brand_key +COMMENT ON VIEW v_brand_store_detail IS 'Quick per-store drill down for a brand with aggregated SKU counts'; +``` + +### 1.9 Feature Flags Table + +```sql +CREATE TABLE feature_flags ( + id SERIAL PRIMARY KEY, + name VARCHAR(100) NOT NULL UNIQUE, + enabled BOOLEAN NOT NULL DEFAULT FALSE, + metadata JSONB DEFAULT '{}', + updated_at TIMESTAMPTZ DEFAULT NOW() +); + +-- Initialize flags +INSERT INTO feature_flags (name, enabled) VALUES + ('INTELLIGENCE_ENABLED', FALSE), + ('INTELLIGENCE_INGEST_ENABLED', FALSE), + ('INTELLIGENCE_API_ENABLED', FALSE), + ('BRAND_METRICS_ENABLED', FALSE); +``` + +--- + +## 2. Ingestion Algorithm + +### 2.1 Product Fingerprinting + +```typescript +import crypto from 'crypto'; + +function generateFingerprint(product: { + name: string; + brand?: string; + variant?: string; + weight?: string; +}): string { + const normalized = [ + (product.name || '').toLowerCase().trim(), + (product.brand || '').toLowerCase().trim(), + (product.variant || '').toLowerCase().trim(), + (product.weight || '').toLowerCase().trim() + ].join('|'); + + return crypto.createHash('sha256').update(normalized).digest('hex').substring(0, 64); +} +``` + +### 2.2 Special Text Parsing + +```typescript +interface ParsedSpecial { + type: 'percent_off' | 'dollar_off' | 'bogo' | 'bundle' | 'set_price' | 'other'; + value: number | null; + raw: string; +} + +function parseSpecialText(text: string | null): ParsedSpecial | null { + if (!text) return null; + + const lower = text.toLowerCase().trim(); + + // 20% off, 25% OFF + const pctMatch = lower.match(/(\d+(?:\.\d+)?)\s*%\s*off/i); + if (pctMatch) { + return { type: 'percent_off', value: parseFloat(pctMatch[1]), raw: text }; + } + + // $5 off, $10 OFF + const dollarMatch = lower.match(/\$(\d+(?:\.\d+)?)\s*off/i); + if (dollarMatch) { + return { type: 'dollar_off', value: parseFloat(dollarMatch[1]), raw: text }; + } + + // BOGO, Buy One Get One + if (/bogo|buy\s*one\s*get\s*one/i.test(lower)) { + return { type: 'bogo', value: 50, raw: text }; // 50% effective discount + } + + // 2 for $50, 3 for $100 + const bundleMatch = lower.match(/(\d+)\s*for\s*\$(\d+(?:\.\d+)?)/i); + if (bundleMatch) { + return { type: 'bundle', value: parseFloat(bundleMatch[2]), raw: text }; + } + + // Now $25, Only $30 + const setPriceMatch = lower.match(/(?:now|only|just)\s*\$(\d+(?:\.\d+)?)/i); + if (setPriceMatch) { + return { type: 'set_price', value: parseFloat(setPriceMatch[1]), raw: text }; + } + + // Has some special but couldn't parse + if (lower.length > 0) { + return { type: 'other', value: null, raw: text }; + } + + return null; +} +``` + +### 2.3 Brand Resolution + +```typescript +async function resolveCanonicalBrand( + db: Pool, + brandName: string | null +): Promise { + if (!brandName || brandName.trim() === '') return null; + + const normalized = brandName.trim(); + + // Try exact match first + const exact = await db.query( + 'SELECT id FROM canonical_brands WHERE LOWER(name) = LOWER($1)', + [normalized] + ); + if (exact.rows.length > 0) return exact.rows[0].id; + + // Try alias match + const alias = await db.query( + 'SELECT id FROM canonical_brands WHERE LOWER($1) = ANY(SELECT LOWER(unnest(aliases)))', + [normalized] + ); + if (alias.rows.length > 0) return alias.rows[0].id; + + // Create new canonical brand + const slug = normalized.toLowerCase() + .replace(/[^a-z0-9]+/g, '-') + .replace(/^-|-$/g, ''); + + const inserted = await db.query( + `INSERT INTO canonical_brands (name, slug) + VALUES ($1, $2) + ON CONFLICT (name) DO UPDATE SET updated_at = NOW() + RETURNING id`, + [normalized, slug] + ); + + return inserted.rows[0].id; +} +``` + +### 2.4 Main Ingestion Function + +```typescript +interface CrawledProduct { + id: number; + name: string; + brand?: string; + variant?: string; + weight?: string; + price: number | null; + in_stock: boolean; + special_text?: string; +} + +interface IngestResult { + crawlRunId: number; + productsProcessed: number; + newProducts: number; + changedProducts: number; + staleProducts: number; + brandsUpdated: number; +} + +async function ingestCrawlResults( + db: Pool, + storeId: number, + dispensaryId: number, + crawledProducts: CrawledProduct[] +): Promise { + const client = await db.connect(); + + try { + await client.query('BEGIN'); + + // Check feature flag + const flagCheck = await client.query( + "SELECT enabled FROM feature_flags WHERE name = 'INTELLIGENCE_INGEST_ENABLED'" + ); + if (!flagCheck.rows[0]?.enabled) { + await client.query('ROLLBACK'); + return { crawlRunId: 0, productsProcessed: 0, newProducts: 0, changedProducts: 0, staleProducts: 0, brandsUpdated: 0 }; + } + + // 1. Create crawl run + const crawlRun = await client.query( + `INSERT INTO crawl_runs (store_id, dispensary_id, status) + VALUES ($1, $2, 'running') RETURNING id`, + [storeId, dispensaryId] + ); + const crawlRunId = crawlRun.rows[0].id; + + // 2. Get existing store_products + const existing = await client.query( + `SELECT id, fingerprint, current_price, current_in_stock, + current_special_text, canonical_brand_id + FROM store_products + WHERE store_id = $1 AND change_status NOT IN ('REMOVED')`, + [storeId] + ); + const existingMap = new Map(existing.rows.map(r => [r.fingerprint, r])); + + // Track seen fingerprints + const seenFingerprints = new Set(); + + let newCount = 0; + let changedCount = 0; + const brandUpdates = new Set(); + + // 3. Process each crawled product + for (const product of crawledProducts) { + const fingerprint = generateFingerprint(product); + seenFingerprints.add(fingerprint); + + const canonicalBrandId = await resolveCanonicalBrand(client, product.brand || null); + const parsedSpecial = parseSpecialText(product.special_text || null); + + const existingProduct = existingMap.get(fingerprint); + + if (!existingProduct) { + // NEW product + const inserted = await client.query( + `INSERT INTO store_products + (store_id, dispensary_id, product_id, canonical_brand_id, fingerprint, + first_seen_at, last_seen_at, current_price, current_in_stock, + current_special_text, current_special_type, current_special_value, + change_status, consecutive_missing) + VALUES ($1, $2, $3, $4, $5, NOW(), NOW(), $6, $7, $8, $9, $10, 'NEW', 0) + RETURNING id`, + [storeId, dispensaryId, product.id, canonicalBrandId, fingerprint, + product.price, product.in_stock, parsedSpecial?.raw, + parsedSpecial?.type, parsedSpecial?.value] + ); + + // Record snapshot + await client.query( + `INSERT INTO store_product_snapshots + (store_product_id, crawl_run_id, price, in_stock, special_text, + special_type, special_value, change_type) + VALUES ($1, $2, $3, $4, $5, $6, $7, 'NEW')`, + [inserted.rows[0].id, crawlRunId, product.price, product.in_stock, + parsedSpecial?.raw, parsedSpecial?.type, parsedSpecial?.value] + ); + + newCount++; + if (canonicalBrandId) brandUpdates.add(canonicalBrandId); + + } else { + // Existing product - check for changes + const changes: Record = {}; + + if (existingProduct.current_price !== product.price) { + changes.price = { old: existingProduct.current_price, new: product.price }; + } + if (existingProduct.current_in_stock !== product.in_stock) { + changes.in_stock = { old: existingProduct.current_in_stock, new: product.in_stock }; + } + if (existingProduct.current_special_text !== (parsedSpecial?.raw || null)) { + changes.special = { old: existingProduct.current_special_text, new: parsedSpecial?.raw }; + } + + const hasChanges = Object.keys(changes).length > 0; + const changeType = hasChanges ? 'CHANGED' : 'UNCHANGED'; + + // Update store_product + await client.query( + `UPDATE store_products SET + last_seen_at = NOW(), + last_changed_at = CASE WHEN $1 THEN NOW() ELSE last_changed_at END, + current_price = $2, + current_in_stock = $3, + current_special_text = $4, + current_special_type = $5, + current_special_value = $6, + change_status = $7, + consecutive_missing = 0, + removed_at = NULL + WHERE id = $8`, + [hasChanges, product.price, product.in_stock, parsedSpecial?.raw, + parsedSpecial?.type, parsedSpecial?.value, changeType, existingProduct.id] + ); + + // Record snapshot + await client.query( + `INSERT INTO store_product_snapshots + (store_product_id, crawl_run_id, price, in_stock, special_text, + special_type, special_value, change_type, changes_detected) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)`, + [existingProduct.id, crawlRunId, product.price, product.in_stock, + parsedSpecial?.raw, parsedSpecial?.type, parsedSpecial?.value, + changeType, hasChanges ? JSON.stringify(changes) : null] + ); + + if (hasChanges) changedCount++; + if (canonicalBrandId) brandUpdates.add(canonicalBrandId); + } + } + + // 4. Mark missing products as STALE/REMOVED + let staleCount = 0; + for (const [fingerprint, existing] of existingMap) { + if (!seenFingerprints.has(fingerprint)) { + const newMissingCount = (existing.consecutive_missing || 0) + 1; + let newStatus: string; + let removedAt: string | null = null; + + if (newMissingCount >= 7) { + newStatus = 'REMOVED'; + removedAt = 'NOW()'; + } else if (newMissingCount >= 3) { + newStatus = 'STALE'; + staleCount++; + } else { + newStatus = existing.change_status; + } + + await client.query( + `UPDATE store_products SET + consecutive_missing = $1, + change_status = $2, + removed_at = ${removedAt || 'removed_at'} + WHERE id = $3`, + [newMissingCount, newStatus, existing.id] + ); + + if (existing.canonical_brand_id) { + brandUpdates.add(existing.canonical_brand_id); + } + } + } + + // 5. Update brand_store_presence for affected brands + for (const brandId of brandUpdates) { + await updateBrandStorePresence(client, brandId, storeId, dispensaryId); + } + + // 6. Finalize crawl run + await client.query( + `UPDATE crawl_runs SET + completed_at = NOW(), + status = 'completed', + products_found = $1, + products_new = $2, + products_changed = $3, + products_stale = $4 + WHERE id = $5`, + [crawledProducts.length, newCount, changedCount, staleCount, crawlRunId] + ); + + await client.query('COMMIT'); + + return { + crawlRunId, + productsProcessed: crawledProducts.length, + newProducts: newCount, + changedProducts: changedCount, + staleProducts: staleCount, + brandsUpdated: brandUpdates.size + }; + + } catch (error) { + await client.query('ROLLBACK'); + throw error; + } finally { + client.release(); + } +} +``` + +### 2.5 Brand Presence Update + +```typescript +async function updateBrandStorePresence( + client: PoolClient, + brandId: number, + storeId: number, + dispensaryId: number +): Promise { + // Count active SKUs for this brand at this store + const counts = await client.query( + `SELECT + COUNT(*) FILTER (WHERE change_status NOT IN ('STALE', 'REMOVED')) as active_count, + COUNT(*) FILTER (WHERE change_status NOT IN ('STALE', 'REMOVED') AND current_in_stock = TRUE) as in_stock_count, + COUNT(*) as total_count + FROM store_products + WHERE canonical_brand_id = $1 AND store_id = $2`, + [brandId, storeId] + ); + + const activeCount = parseInt(counts.rows[0].active_count); + const inStockCount = parseInt(counts.rows[0].in_stock_count); + const totalCount = parseInt(counts.rows[0].total_count); + + // Check if presence record exists + const existing = await client.query( + `SELECT id, is_active FROM brand_store_presence + WHERE canonical_brand_id = $1 AND store_id = $2`, + [brandId, storeId] + ); + + if (existing.rows.length === 0) { + // New presence - brand just appeared at store + await client.query( + `INSERT INTO brand_store_presence + (canonical_brand_id, store_id, dispensary_id, first_seen_at, last_seen_at, + active_sku_count, in_stock_sku_count, total_sku_count, is_active) + VALUES ($1, $2, $3, NOW(), NOW(), $4, $5, $6, TRUE)`, + [brandId, storeId, dispensaryId, activeCount, inStockCount, totalCount] + ); + + // Record ADDED event + await client.query( + `INSERT INTO brand_store_events + (canonical_brand_id, store_id, dispensary_id, event_type, event_date, sku_count) + VALUES ($1, $2, $3, 'ADDED', CURRENT_DATE, $4)`, + [brandId, storeId, dispensaryId, activeCount] + ); + + } else { + const wasActive = existing.rows[0].is_active; + const isNowActive = activeCount > 0; + + // Update presence + await client.query( + `UPDATE brand_store_presence SET + last_seen_at = CASE WHEN $1 > 0 THEN NOW() ELSE last_seen_at END, + removed_at = CASE WHEN $1 = 0 THEN NOW() ELSE NULL END, + active_sku_count = $1, + in_stock_sku_count = $2, + total_sku_count = $3, + is_active = $4, + consecutive_missing = CASE WHEN $1 > 0 THEN 0 ELSE consecutive_missing + 1 END + WHERE id = $5`, + [activeCount, inStockCount, totalCount, isNowActive, existing.rows[0].id] + ); + + // Record events for status changes + if (wasActive && !isNowActive) { + await client.query( + `INSERT INTO brand_store_events + (canonical_brand_id, store_id, dispensary_id, event_type, event_date) + VALUES ($1, $2, $3, 'REMOVED', CURRENT_DATE)`, + [brandId, storeId, dispensaryId] + ); + } else if (!wasActive && isNowActive) { + await client.query( + `INSERT INTO brand_store_events + (canonical_brand_id, store_id, dispensary_id, event_type, event_date, sku_count) + VALUES ($1, $2, $3, 'REAPPEARED', CURRENT_DATE, $4)`, + [brandId, storeId, dispensaryId, activeCount] + ); + } + } +} +``` + +--- + +## 3. Daily Aggregation Job + +```typescript +async function runDailyBrandAggregation(db: Pool): Promise { + const client = await db.connect(); + + try { + await client.query('BEGIN'); + + const today = new Date().toISOString().split('T')[0]; + + // Get all brands with cannabrands_id (active in Cannabrands platform) + const brands = await client.query( + `SELECT id FROM canonical_brands WHERE cannabrands_id IS NOT NULL` + ); + + for (const brand of brands.rows) { + const brandId = brand.id; + + // Compute daily metrics + const metrics = await client.query(` + SELECT + COUNT(DISTINCT bsp.store_id) FILTER (WHERE bsp.is_active) as total_stores, + COUNT(DISTINCT bse.store_id) FILTER (WHERE bse.event_type = 'ADDED' AND bse.event_date = $2) as stores_added, + COUNT(DISTINCT bse.store_id) FILTER (WHERE bse.event_type = 'REMOVED' AND bse.event_date = $2) as stores_removed, + COUNT(sp.id) FILTER (WHERE sp.change_status NOT IN ('STALE', 'REMOVED')) as total_skus, + COUNT(sp.id) FILTER (WHERE sp.change_status NOT IN ('STALE', 'REMOVED') AND sp.current_in_stock) as skus_in_stock, + COUNT(sp.id) FILTER (WHERE sp.change_status = 'NEW' AND DATE(sp.first_seen_at) = $2) as new_skus, + COUNT(sp.id) FILTER (WHERE sp.change_status = 'REMOVED' AND DATE(sp.removed_at) = $2) as removed_skus, + AVG(sp.current_price) FILTER (WHERE sp.change_status NOT IN ('STALE', 'REMOVED') AND sp.current_price IS NOT NULL) as avg_price, + MIN(sp.current_price) FILTER (WHERE sp.change_status NOT IN ('STALE', 'REMOVED') AND sp.current_price IS NOT NULL) as min_price, + MAX(sp.current_price) FILTER (WHERE sp.change_status NOT IN ('STALE', 'REMOVED') AND sp.current_price IS NOT NULL) as max_price, + COUNT(sp.id) FILTER (WHERE sp.current_special_type IS NOT NULL AND sp.change_status NOT IN ('STALE', 'REMOVED')) as skus_on_promo, + COALESCE(SUM(sp.current_special_value) FILTER (WHERE sp.current_special_type IS NOT NULL), 0) as promo_value_sum + FROM canonical_brands cb + LEFT JOIN store_products sp ON sp.canonical_brand_id = cb.id + LEFT JOIN brand_store_presence bsp ON bsp.canonical_brand_id = cb.id + LEFT JOIN brand_store_events bse ON bse.canonical_brand_id = cb.id + WHERE cb.id = $1 + GROUP BY cb.id + `, [brandId, today]); + + const m = metrics.rows[0] || {}; + + // Upsert daily metrics + await client.query(` + INSERT INTO brand_daily_metrics + (canonical_brand_id, date, total_stores, stores_added, stores_removed, + total_skus, skus_in_stock, new_skus, removed_skus, + avg_price, min_price, max_price, price_changes, skus_on_promo, promo_value_sum) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, 0, $13, $14) + ON CONFLICT (canonical_brand_id, date) DO UPDATE SET + total_stores = EXCLUDED.total_stores, + stores_added = EXCLUDED.stores_added, + stores_removed = EXCLUDED.stores_removed, + total_skus = EXCLUDED.total_skus, + skus_in_stock = EXCLUDED.skus_in_stock, + new_skus = EXCLUDED.new_skus, + removed_skus = EXCLUDED.removed_skus, + avg_price = EXCLUDED.avg_price, + min_price = EXCLUDED.min_price, + max_price = EXCLUDED.max_price, + skus_on_promo = EXCLUDED.skus_on_promo, + promo_value_sum = EXCLUDED.promo_value_sum, + computed_at = NOW() + `, [brandId, today, m.total_stores || 0, m.stores_added || 0, m.stores_removed || 0, + m.total_skus || 0, m.skus_in_stock || 0, m.new_skus || 0, m.removed_skus || 0, + m.avg_price, m.min_price, m.max_price, m.skus_on_promo || 0, m.promo_value_sum || 0]); + + // Compute promo metrics + const promoMetrics = await client.query(` + SELECT + COUNT(*) as total_promos, + COUNT(*) FILTER (WHERE current_special_type = 'percent_off') as percent_off_count, + COUNT(*) FILTER (WHERE current_special_type = 'bogo') as bogo_count, + COUNT(*) FILTER (WHERE current_special_type = 'bundle') as bundle_count, + COUNT(*) FILTER (WHERE current_special_type NOT IN ('percent_off', 'bogo', 'bundle')) as other_count, + AVG(current_special_value) FILTER (WHERE current_special_type = 'percent_off') as avg_discount_pct, + COUNT(DISTINCT store_id) as stores_with_promos + FROM store_products + WHERE canonical_brand_id = $1 + AND change_status NOT IN ('STALE', 'REMOVED') + AND current_special_type IS NOT NULL + `, [brandId]); + + const p = promoMetrics.rows[0] || {}; + + await client.query(` + INSERT INTO brand_promo_daily_metrics + (canonical_brand_id, date, total_promos, percent_off_count, bogo_count, + bundle_count, other_count, avg_discount_pct, stores_with_promos) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) + ON CONFLICT (canonical_brand_id, date) DO UPDATE SET + total_promos = EXCLUDED.total_promos, + percent_off_count = EXCLUDED.percent_off_count, + bogo_count = EXCLUDED.bogo_count, + bundle_count = EXCLUDED.bundle_count, + other_count = EXCLUDED.other_count, + avg_discount_pct = EXCLUDED.avg_discount_pct, + stores_with_promos = EXCLUDED.stores_with_promos, + computed_at = NOW() + `, [brandId, today, p.total_promos || 0, p.percent_off_count || 0, p.bogo_count || 0, + p.bundle_count || 0, p.other_count || 0, p.avg_discount_pct, p.stores_with_promos || 0]); + } + + await client.query('COMMIT'); + console.log(`Daily aggregation completed for ${brands.rows.length} brands`); + + } catch (error) { + await client.query('ROLLBACK'); + throw error; + } finally { + client.release(); + } +} +``` + +### 3.2 Historical Backfill Script + +The backfill script must be **idempotent** - re-running it for the same date range should not duplicate or skew data. + +```typescript +// scripts/backfill-brand-metrics.ts + +interface BackfillOptions { + startDate: string; // YYYY-MM-DD + endDate: string; // YYYY-MM-DD + brandIds?: number[]; // Optional: specific brands to backfill +} + +async function backfillBrandMetrics( + db: Pool, + options: BackfillOptions +): Promise { + const { startDate, endDate, brandIds } = options; + + // Generate date range + const dates: string[] = []; + let current = new Date(startDate); + const end = new Date(endDate); + + while (current <= end) { + dates.push(current.toISOString().split('T')[0]); + current.setDate(current.getDate() + 1); + } + + console.log(`Backfilling ${dates.length} days from ${startDate} to ${endDate}`); + + // Get brands to process + let brandsQuery = `SELECT id FROM canonical_brands WHERE cannabrands_id IS NOT NULL`; + const brandsParams: any[] = []; + + if (brandIds && brandIds.length > 0) { + brandsQuery += ` AND id = ANY($1)`; + brandsParams.push(brandIds); + } + + const brands = await db.query(brandsQuery, brandsParams); + + for (const targetDate of dates) { + console.log(`Processing ${targetDate}...`); + + for (const brand of brands.rows) { + await aggregateBrandDailyMetrics(db, brand.id, targetDate); + } + } + + console.log('Backfill complete'); +} + +/** + * Aggregate metrics for a single brand on a single date. + * IDEMPOTENT: Uses INSERT ... ON CONFLICT DO UPDATE to ensure + * re-running for the same (brand_id, date) replaces rather than duplicates. + */ +async function aggregateBrandDailyMetrics( + db: Pool, + brandId: number, + targetDate: string +): Promise { + const client = await db.connect(); + + try { + await client.query('BEGIN'); + + // Compute metrics for this brand on this date + const metrics = await client.query(` + SELECT + COUNT(DISTINCT bsp.store_id) FILTER (WHERE bsp.is_active) as total_stores, + COUNT(DISTINCT bse.store_id) FILTER (WHERE bse.event_type = 'ADDED' AND bse.event_date = $2::date) as stores_added, + COUNT(DISTINCT bse.store_id) FILTER (WHERE bse.event_type = 'REMOVED' AND bse.event_date = $2::date) as stores_removed, + COUNT(sp.id) FILTER (WHERE sp.change_status NOT IN ('STALE', 'REMOVED')) as total_skus, + COUNT(sp.id) FILTER (WHERE sp.change_status NOT IN ('STALE', 'REMOVED') AND sp.current_in_stock) as skus_in_stock, + COUNT(sp.id) FILTER (WHERE sp.change_status = 'NEW' AND DATE(sp.first_seen_at) = $2::date) as new_skus, + COUNT(sp.id) FILTER (WHERE sp.change_status = 'REMOVED' AND DATE(sp.removed_at) = $2::date) as removed_skus, + AVG(sp.current_price) FILTER (WHERE sp.change_status NOT IN ('STALE', 'REMOVED') AND sp.current_price IS NOT NULL) as avg_price, + MIN(sp.current_price) FILTER (WHERE sp.change_status NOT IN ('STALE', 'REMOVED') AND sp.current_price IS NOT NULL) as min_price, + MAX(sp.current_price) FILTER (WHERE sp.change_status NOT IN ('STALE', 'REMOVED') AND sp.current_price IS NOT NULL) as max_price, + COUNT(sp.id) FILTER (WHERE sp.current_special_type IS NOT NULL AND sp.change_status NOT IN ('STALE', 'REMOVED')) as skus_on_promo, + COALESCE(SUM(sp.current_special_value) FILTER (WHERE sp.current_special_type IS NOT NULL), 0) as promo_value_sum + FROM canonical_brands cb + LEFT JOIN store_products sp ON sp.canonical_brand_id = cb.id + LEFT JOIN brand_store_presence bsp ON bsp.canonical_brand_id = cb.id + LEFT JOIN brand_store_events bse ON bse.canonical_brand_id = cb.id + WHERE cb.id = $1 + GROUP BY cb.id + `, [brandId, targetDate]); + + const m = metrics.rows[0] || {}; + + // IDEMPOTENT: INSERT ... ON CONFLICT DO UPDATE + // This ensures re-running backfill replaces existing data rather than duplicating + await client.query(` + INSERT INTO brand_daily_metrics + (canonical_brand_id, date, total_stores, stores_added, stores_removed, + total_skus, skus_in_stock, new_skus, removed_skus, + avg_price, min_price, max_price, price_changes, skus_on_promo, promo_value_sum, computed_at) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, 0, $13, $14, NOW()) + ON CONFLICT (canonical_brand_id, date) DO UPDATE SET + total_stores = EXCLUDED.total_stores, + stores_added = EXCLUDED.stores_added, + stores_removed = EXCLUDED.stores_removed, + total_skus = EXCLUDED.total_skus, + skus_in_stock = EXCLUDED.skus_in_stock, + new_skus = EXCLUDED.new_skus, + removed_skus = EXCLUDED.removed_skus, + avg_price = EXCLUDED.avg_price, + min_price = EXCLUDED.min_price, + max_price = EXCLUDED.max_price, + skus_on_promo = EXCLUDED.skus_on_promo, + promo_value_sum = EXCLUDED.promo_value_sum, + computed_at = NOW() + `, [brandId, targetDate, m.total_stores || 0, m.stores_added || 0, m.stores_removed || 0, + m.total_skus || 0, m.skus_in_stock || 0, m.new_skus || 0, m.removed_skus || 0, + m.avg_price, m.min_price, m.max_price, m.skus_on_promo || 0, m.promo_value_sum || 0]); + + // Same for promo metrics - IDEMPOTENT + const promoMetrics = await client.query(` + SELECT + COUNT(*) as total_promos, + COUNT(*) FILTER (WHERE current_special_type = 'percent_off') as percent_off_count, + COUNT(*) FILTER (WHERE current_special_type = 'bogo') as bogo_count, + COUNT(*) FILTER (WHERE current_special_type = 'bundle') as bundle_count, + COUNT(*) FILTER (WHERE current_special_type NOT IN ('percent_off', 'bogo', 'bundle')) as other_count, + AVG(current_special_value) FILTER (WHERE current_special_type = 'percent_off') as avg_discount_pct, + COUNT(DISTINCT store_id) as stores_with_promos + FROM store_products + WHERE canonical_brand_id = $1 + AND change_status NOT IN ('STALE', 'REMOVED') + AND current_special_type IS NOT NULL + `, [brandId]); + + const p = promoMetrics.rows[0] || {}; + + await client.query(` + INSERT INTO brand_promo_daily_metrics + (canonical_brand_id, date, total_promos, percent_off_count, bogo_count, + bundle_count, other_count, avg_discount_pct, stores_with_promos, computed_at) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, NOW()) + ON CONFLICT (canonical_brand_id, date) DO UPDATE SET + total_promos = EXCLUDED.total_promos, + percent_off_count = EXCLUDED.percent_off_count, + bogo_count = EXCLUDED.bogo_count, + bundle_count = EXCLUDED.bundle_count, + other_count = EXCLUDED.other_count, + avg_discount_pct = EXCLUDED.avg_discount_pct, + stores_with_promos = EXCLUDED.stores_with_promos, + computed_at = NOW() + `, [brandId, targetDate, p.total_promos || 0, p.percent_off_count || 0, p.bogo_count || 0, + p.bundle_count || 0, p.other_count || 0, p.avg_discount_pct, p.stores_with_promos || 0]); + + await client.query('COMMIT'); + + } catch (error) { + await client.query('ROLLBACK'); + throw error; + } finally { + client.release(); + } +} + +// Usage: +// npx tsx scripts/backfill-brand-metrics.ts --start 2025-01-01 --end 2025-01-31 +``` + +**Idempotency Guarantees:** +- Uses `INSERT ... ON CONFLICT (canonical_brand_id, date) DO UPDATE` for all upserts +- Re-running backfill for the same date range will replace existing data, not duplicate it +- Safe to run multiple times if interrupted or to refresh stale data + +--- + +## 4. API Design + +### 4.1 Brand Key Usage for Cannabrands App + +The `brand_key` parameter in all API endpoints maps directly to `canonical_brands.cannabrands_id`. + +**How it works:** + +1. The Cannabrands app stores the `cannabrands_id` for each brand in their system +2. When calling intelligence endpoints, they pass this as the `:brand_key` URL parameter +3. Our API resolves `brand_key` → `canonical_brands.id` for all subsequent queries +4. All data is scoped by this resolved `canonical_brand_id` + +**Resolution Flow:** +``` +API Request: GET /api/brands/raw-garden/intelligence/summary + ↑ + brand_key + ↓ +SELECT id FROM canonical_brands WHERE cannabrands_id = 'raw-garden' + ↓ + canonical_brand_id = 42 + ↓ + All queries scoped: WHERE canonical_brand_id = 42 +``` + +**For the Cannabrands App Team:** +- Store `cannabrands_id` as your brand identifier for this API +- Use `cannabrands_slug` for display/URL purposes only +- The admin mapping UI will set both `cannabrands_id` and `cannabrands_slug` +- JWT tokens must include a `brand_key` claim matching the `cannabrands_id` + +### 4.2 Feature Flag Middleware + +Feature flags provide zero-code-change rollback capability. Check flags at both ingestion and API layers. + +```typescript +import { getConfig } from '../config'; + +// Cache feature flags for 60 seconds to reduce DB load +let flagCache: Map = new Map(); +const FLAG_CACHE_TTL = 60000; // 60 seconds + +async function getFeatureFlag(db: Pool, name: string): Promise { + const cached = flagCache.get(name); + if (cached && cached.expires > Date.now()) { + return cached.value; + } + + // Check environment variable override first + const envOverride = process.env[name]; + if (envOverride !== undefined) { + const value = envOverride.toLowerCase() === 'true'; + flagCache.set(name, { value, expires: Date.now() + FLAG_CACHE_TTL }); + return value; + } + + // Check database + const result = await db.query( + 'SELECT enabled FROM feature_flags WHERE name = $1', + [name] + ); + + const value = result.rows[0]?.enabled ?? false; + flagCache.set(name, { value, expires: Date.now() + FLAG_CACHE_TTL }); + return value; +} + +// API Middleware - returns 404 when disabled (hides feature existence) +async function requireIntelligenceEnabled(req: Request, res: Response, next: NextFunction) { + const enabled = await getFeatureFlag(db, 'INTELLIGENCE_API_ENABLED'); + + if (!enabled) { + return res.status(404).json({ + error: 'Brand intelligence is not enabled', + code: 'FEATURE_NOT_FOUND' + }); + } + + next(); +} + +// Ingestion Guard - silently skips when disabled +async function isIngestEnabled(db: Pool): Promise { + return getFeatureFlag(db, 'INTELLIGENCE_INGEST_ENABLED'); +} + +// Aggregation Guard - silently skips when disabled +async function isAggregationEnabled(db: Pool): Promise { + return getFeatureFlag(db, 'BRAND_METRICS_ENABLED'); +} +``` + +**Usage in Ingestion:** +```typescript +// In crawl completion handler +async function onCrawlComplete(storeId: number, dispensaryId: number, products: Product[]) { + // Guard: check if intelligence ingestion is enabled + if (!await isIngestEnabled(db)) { + console.log('Intelligence ingest is disabled; skipping intelligence processing.'); + return; + } + + // Proceed with intelligence ingestion + await ingestCrawlResults(db, storeId, dispensaryId, products); +} +``` + +**Usage in Aggregation Job:** +```typescript +// In daily cron job +async function runDailyAggregationJob() { + // Guard: check if aggregation is enabled + if (!await isAggregationEnabled(db)) { + console.log('Brand metrics aggregation is disabled; skipping daily aggregation.'); + return; + } + + // Proceed with aggregation + await runDailyBrandAggregation(db); +} +``` + +### 4.3 Brand Authorization Middleware + +```typescript +interface BrandAuthPayload { + brand_key: string; // cannabrands_id + permissions: string[]; +} + +async function requireBrandAccess(req: Request, res: Response, next: NextFunction) { + const { brand_key } = req.params; + const auth = req.user as BrandAuthPayload; + + if (!auth || auth.brand_key !== brand_key) { + return res.status(403).json({ + error: 'Access denied to this brand', + code: 'BRAND_ACCESS_DENIED' + }); + } + + // Resolve canonical_brand_id + const brand = await db.query( + 'SELECT id FROM canonical_brands WHERE cannabrands_id = $1', + [brand_key] + ); + + if (brand.rows.length === 0) { + return res.status(404).json({ + error: 'Brand not found', + code: 'BRAND_NOT_FOUND' + }); + } + + req.canonicalBrandId = brand.rows[0].id; + next(); +} +``` + +### 4.3 Brand Intelligence Endpoints + +#### GET /api/brands/:brand_key/intelligence/summary + +Returns current summary metrics for a brand. + +```typescript +router.get('/brands/:brand_key/intelligence/summary', + requireIntelligenceEnabled, + requireBrandAccess, + async (req, res) => { + const brandId = req.canonicalBrandId; + + // Get latest daily metrics + const latest = await db.query(` + SELECT * FROM brand_daily_metrics + WHERE canonical_brand_id = $1 + ORDER BY date DESC LIMIT 1 + `, [brandId]); + + // Get 7-day and 30-day comparisons + const comparison = await db.query(` + SELECT + date, + total_stores, + total_skus, + skus_in_stock, + skus_on_promo + FROM brand_daily_metrics + WHERE canonical_brand_id = $1 + AND date IN (CURRENT_DATE, CURRENT_DATE - 7, CURRENT_DATE - 30) + ORDER BY date DESC + `, [brandId]); + + const today = comparison.rows.find(r => r.date === new Date().toISOString().split('T')[0]); + const week = comparison.rows.find(r => { + const d = new Date(); + d.setDate(d.getDate() - 7); + return r.date === d.toISOString().split('T')[0]; + }); + const month = comparison.rows.find(r => { + const d = new Date(); + d.setDate(d.getDate() - 30); + return r.date === d.toISOString().split('T')[0]; + }); + + res.json({ + brand_key: req.params.brand_key, + as_of: new Date().toISOString(), + current: { + total_stores: today?.total_stores || 0, + total_skus: today?.total_skus || 0, + skus_in_stock: today?.skus_in_stock || 0, + skus_on_promo: today?.skus_on_promo || 0, + avg_price: latest.rows[0]?.avg_price || null, + price_range: { + min: latest.rows[0]?.min_price || null, + max: latest.rows[0]?.max_price || null + } + }, + changes: { + week: { + stores: (today?.total_stores || 0) - (week?.total_stores || 0), + skus: (today?.total_skus || 0) - (week?.total_skus || 0) + }, + month: { + stores: (today?.total_stores || 0) - (month?.total_stores || 0), + skus: (today?.total_skus || 0) - (month?.total_skus || 0) + } + } + }); + } +); +``` + +**Response Example:** +```json +{ + "brand_key": "raw-garden", + "as_of": "2025-01-15T12:00:00Z", + "current": { + "total_stores": 127, + "total_skus": 543, + "skus_in_stock": 489, + "skus_on_promo": 34, + "avg_price": 42.50, + "price_range": { "min": 25.00, "max": 85.00 } + }, + "changes": { + "week": { "stores": 3, "skus": 12 }, + "month": { "stores": 15, "skus": 67 } + } +} +``` + +#### GET /api/brands/:brand_key/intelligence/store-timeseries + +Returns daily store/SKU counts for charting. + +```typescript +router.get('/brands/:brand_key/intelligence/store-timeseries', + requireIntelligenceEnabled, + requireBrandAccess, + async (req, res) => { + const brandId = req.canonicalBrandId; + const days = Math.min(parseInt(req.query.days as string) || 30, 365); + + const data = await db.query(` + SELECT + date, + total_stores, + stores_added, + stores_removed, + total_skus, + skus_in_stock, + new_skus, + removed_skus + FROM brand_daily_metrics + WHERE canonical_brand_id = $1 + AND date >= CURRENT_DATE - $2 + ORDER BY date ASC + `, [brandId, days]); + + res.json({ + brand_key: req.params.brand_key, + period: { days, start: data.rows[0]?.date, end: data.rows[data.rows.length - 1]?.date }, + timeseries: data.rows.map(r => ({ + date: r.date, + stores: { + total: r.total_stores, + added: r.stores_added, + removed: r.stores_removed + }, + skus: { + total: r.total_skus, + in_stock: r.skus_in_stock, + new: r.new_skus, + removed: r.removed_skus + } + })) + }); + } +); +``` + +**Response Example:** +```json +{ + "brand_key": "raw-garden", + "period": { "days": 30, "start": "2024-12-16", "end": "2025-01-15" }, + "timeseries": [ + { + "date": "2024-12-16", + "stores": { "total": 112, "added": 2, "removed": 0 }, + "skus": { "total": 476, "in_stock": 423, "new": 5, "removed": 1 } + }, + { + "date": "2024-12-17", + "stores": { "total": 114, "added": 2, "removed": 0 }, + "skus": { "total": 482, "in_stock": 430, "new": 8, "removed": 2 } + } + ] +} +``` + +#### GET /api/brands/:brand_key/intelligence/store-changes + +Returns recent store addition/removal events. + +```typescript +router.get('/brands/:brand_key/intelligence/store-changes', + requireIntelligenceEnabled, + requireBrandAccess, + async (req, res) => { + const brandId = req.canonicalBrandId; + const limit = Math.min(parseInt(req.query.limit as string) || 50, 200); + const eventType = req.query.event_type as string; // 'ADDED', 'REMOVED', 'REAPPEARED' + + let query = ` + SELECT + bse.event_type, + bse.event_date, + bse.sku_count, + s.name as store_name, + s.slug as store_slug, + d.name as dispensary_name, + d.city, + d.state + FROM brand_store_events bse + JOIN stores s ON s.id = bse.store_id + JOIN dispensaries d ON d.id = bse.dispensary_id + WHERE bse.canonical_brand_id = $1 + `; + const params: any[] = [brandId]; + + if (eventType) { + query += ` AND bse.event_type = $2`; + params.push(eventType); + } + + query += ` ORDER BY bse.event_date DESC, bse.created_at DESC LIMIT $${params.length + 1}`; + params.push(limit); + + const data = await db.query(query, params); + + res.json({ + brand_key: req.params.brand_key, + events: data.rows.map(r => ({ + event_type: r.event_type, + date: r.event_date, + store: { + name: r.store_name, + slug: r.store_slug + }, + dispensary: { + name: r.dispensary_name, + location: `${r.city}, ${r.state}` + }, + sku_count: r.sku_count + })) + }); + } +); +``` + +**Response Example:** +```json +{ + "brand_key": "raw-garden", + "events": [ + { + "event_type": "ADDED", + "date": "2025-01-14", + "store": { "name": "Green Thumb LA", "slug": "green-thumb-la" }, + "dispensary": { "name": "Green Thumb Dispensary", "location": "Los Angeles, CA" }, + "sku_count": 12 + }, + { + "event_type": "REMOVED", + "date": "2025-01-12", + "store": { "name": "MedMen Venice", "slug": "medmen-venice" }, + "dispensary": { "name": "MedMen", "location": "Venice, CA" }, + "sku_count": null + } + ] +} +``` + +#### GET /api/brands/:brand_key/intelligence/promo-timeseries + +Returns daily promo/deal statistics. + +```typescript +router.get('/brands/:brand_key/intelligence/promo-timeseries', + requireIntelligenceEnabled, + requireBrandAccess, + async (req, res) => { + const brandId = req.canonicalBrandId; + const days = Math.min(parseInt(req.query.days as string) || 30, 365); + + const data = await db.query(` + SELECT + date, + total_promos, + percent_off_count, + bogo_count, + bundle_count, + other_count, + avg_discount_pct, + stores_with_promos + FROM brand_promo_daily_metrics + WHERE canonical_brand_id = $1 + AND date >= CURRENT_DATE - $2 + ORDER BY date ASC + `, [brandId, days]); + + res.json({ + brand_key: req.params.brand_key, + period: { days }, + timeseries: data.rows.map(r => ({ + date: r.date, + total_promos: r.total_promos, + breakdown: { + percent_off: r.percent_off_count, + bogo: r.bogo_count, + bundle: r.bundle_count, + other: r.other_count + }, + avg_discount_pct: r.avg_discount_pct, + stores_with_promos: r.stores_with_promos + })) + }); + } +); +``` + +#### GET /api/brands/:brand_key/intelligence/stores + +Returns all stores carrying this brand with current metrics. + +```typescript +router.get('/brands/:brand_key/intelligence/stores', + requireIntelligenceEnabled, + requireBrandAccess, + async (req, res) => { + const brandId = req.canonicalBrandId; + const activeOnly = req.query.active !== 'false'; + + let query = ` + SELECT + bsp.is_active, + bsp.first_seen_at, + bsp.last_seen_at, + bsp.removed_at, + bsp.active_sku_count, + bsp.in_stock_sku_count, + s.name as store_name, + s.slug as store_slug, + d.name as dispensary_name, + d.city, + d.state, + d.dutchie_url + FROM brand_store_presence bsp + JOIN stores s ON s.id = bsp.store_id + JOIN dispensaries d ON d.id = bsp.dispensary_id + WHERE bsp.canonical_brand_id = $1 + `; + + if (activeOnly) { + query += ` AND bsp.is_active = TRUE`; + } + + query += ` ORDER BY bsp.active_sku_count DESC, s.name ASC`; + + const data = await db.query(query, [brandId]); + + res.json({ + brand_key: req.params.brand_key, + total_stores: data.rows.length, + stores: data.rows.map(r => ({ + store: { + name: r.store_name, + slug: r.store_slug + }, + dispensary: { + name: r.dispensary_name, + location: `${r.city}, ${r.state}`, + dutchie_url: r.dutchie_url + }, + presence: { + is_active: r.is_active, + first_seen: r.first_seen_at, + last_seen: r.last_seen_at, + removed_at: r.removed_at + }, + skus: { + active: r.active_sku_count, + in_stock: r.in_stock_sku_count + } + })) + }); + } +); +``` + +#### GET /api/brands/:brand_key/intelligence/products + +Returns all products for this brand with lifecycle data. + +```typescript +router.get('/brands/:brand_key/intelligence/products', + requireIntelligenceEnabled, + requireBrandAccess, + async (req, res) => { + const brandId = req.canonicalBrandId; + const storeSlug = req.query.store as string; + const status = req.query.status as string; // NEW, CHANGED, STALE, REMOVED + const limit = Math.min(parseInt(req.query.limit as string) || 100, 500); + const offset = parseInt(req.query.offset as string) || 0; + + let query = ` + SELECT + sp.id, + sp.fingerprint, + sp.first_seen_at, + sp.last_seen_at, + sp.last_changed_at, + sp.removed_at, + sp.current_price, + sp.current_in_stock, + sp.current_special_text, + sp.current_special_type, + sp.current_special_value, + sp.change_status, + p.name as product_name, + p.variant, + p.weight, + s.name as store_name, + s.slug as store_slug, + d.city, + d.state + FROM store_products sp + JOIN products p ON p.id = sp.product_id + JOIN stores s ON s.id = sp.store_id + JOIN dispensaries d ON d.id = sp.dispensary_id + WHERE sp.canonical_brand_id = $1 + `; + const params: any[] = [brandId]; + let paramIndex = 2; + + if (storeSlug) { + query += ` AND s.slug = $${paramIndex}`; + params.push(storeSlug); + paramIndex++; + } + + if (status) { + query += ` AND sp.change_status = $${paramIndex}`; + params.push(status); + paramIndex++; + } + + query += ` ORDER BY sp.last_seen_at DESC LIMIT $${paramIndex} OFFSET $${paramIndex + 1}`; + params.push(limit, offset); + + const data = await db.query(query, params); + + // Get total count + let countQuery = ` + SELECT COUNT(*) FROM store_products sp + JOIN stores s ON s.id = sp.store_id + WHERE sp.canonical_brand_id = $1 + `; + const countParams: any[] = [brandId]; + + if (storeSlug) { + countQuery += ` AND s.slug = $2`; + countParams.push(storeSlug); + } + if (status) { + countQuery += ` AND sp.change_status = $${countParams.length + 1}`; + countParams.push(status); + } + + const countResult = await db.query(countQuery, countParams); + + res.json({ + brand_key: req.params.brand_key, + total: parseInt(countResult.rows[0].count), + limit, + offset, + products: data.rows.map(r => ({ + id: r.id, + fingerprint: r.fingerprint, + name: r.product_name, + variant: r.variant, + weight: r.weight, + store: { + name: r.store_name, + slug: r.store_slug, + location: `${r.city}, ${r.state}` + }, + lifecycle: { + status: r.change_status, + first_seen: r.first_seen_at, + last_seen: r.last_seen_at, + last_changed: r.last_changed_at, + removed_at: r.removed_at + }, + current: { + price: r.current_price, + in_stock: r.current_in_stock, + special: r.current_special_text ? { + text: r.current_special_text, + type: r.current_special_type, + value: r.current_special_value + } : null + } + })) + }); + } +); +``` + +--- + +## 5. Rollback & Feature Flag Strategy + +**Key Principle:** Normal rollback = flip feature flags, NOT drop tables or modify code. + +Feature flags provide **zero-code-change rollback capability**. All intelligence features are guarded by flags that can be toggled via database update or environment variable override. + +### 5.1 Feature Flags + +| Flag Name | Purpose | Default | Scope | +|-----------|---------|---------|-------| +| `INTELLIGENCE_ENABLED` | Master kill switch for entire intelligence layer | `FALSE` | All | +| `INTELLIGENCE_INGEST_ENABLED` | Controls whether crawl results populate intelligence tables | `FALSE` | Ingestion | +| `INTELLIGENCE_API_ENABLED` | Controls whether intelligence API endpoints respond | `FALSE` | API | +| `BRAND_METRICS_ENABLED` | Controls whether daily aggregation job runs | `FALSE` | Aggregation | + +**Flag Resolution Order:** +1. Environment variable (highest priority): `INTELLIGENCE_API_ENABLED=false` +2. Database `feature_flags` table +3. Default value (FALSE) + +### 5.2 Rollback Procedure (Zero-Code-Change) + +**Level 1: Disable API (immediate)** +```sql +UPDATE feature_flags SET enabled = FALSE WHERE name = 'INTELLIGENCE_API_ENABLED'; +``` +- Intelligence API returns 503 +- Existing WordPress plugin unaffected +- Data continues to be ingested + +**Level 2: Disable Ingestion (stop data flow)** +```sql +UPDATE feature_flags SET enabled = FALSE WHERE name = 'INTELLIGENCE_INGEST_ENABLED'; +``` +- Crawl results no longer populate intelligence tables +- Existing data preserved +- API still available if enabled + +**Level 3: Disable Aggregation (stop brand metrics)** +```sql +UPDATE feature_flags SET enabled = FALSE WHERE name = 'BRAND_METRICS_ENABLED'; +``` +- Daily aggregation job skips execution +- Historical aggregated data preserved + +**Level 4: Full Disable (all features off)** +```sql +UPDATE feature_flags SET enabled = FALSE WHERE name = 'INTELLIGENCE_ENABLED'; +``` +- Master switch that disables everything + +### 5.3 Recovery Procedure + +```sql +-- Re-enable in order +UPDATE feature_flags SET enabled = TRUE WHERE name = 'INTELLIGENCE_ENABLED'; +UPDATE feature_flags SET enabled = TRUE WHERE name = 'INTELLIGENCE_INGEST_ENABLED'; +UPDATE feature_flags SET enabled = TRUE WHERE name = 'BRAND_METRICS_ENABLED'; +UPDATE feature_flags SET enabled = TRUE WHERE name = 'INTELLIGENCE_API_ENABLED'; +``` + +### 5.4 Data Cleanup (Nuclear Option) + +Only if absolutely necessary - preserves schema but clears data: + +```sql +-- Clear all intelligence data (preserves schema) +TRUNCATE brand_promo_daily_metrics CASCADE; +TRUNCATE brand_daily_metrics CASCADE; +TRUNCATE brand_store_events CASCADE; +TRUNCATE brand_store_presence CASCADE; +TRUNCATE store_product_snapshots CASCADE; +TRUNCATE store_products CASCADE; +TRUNCATE crawl_runs CASCADE; +-- Keep canonical_brands as they may be linked to Cannabrands +``` + +--- + +## 6. Implementation Plan + +### Phase 1: Foundation (Week 1) + +1. **Create migration file** with all new tables, ENUMs, and indexes +2. **Add feature_flags table** and initialize all flags to FALSE +3. **Implement fingerprint generation** and special text parsing utilities +4. **Create canonical_brands seed** with initial brand data +5. **Deploy migration** to staging environment + +### Phase 2: Ingestion Layer (Week 2) + +1. **Implement brand resolution** function +2. **Implement main ingestion function** (`ingestCrawlResults`) +3. **Add ingestion hook** to existing crawl completion handler +4. **Test with single store** in staging +5. **Enable `INTELLIGENCE_INGEST_ENABLED`** in staging + +### Phase 3: Brand Presence Tracking (Week 3) + +1. **Implement brand presence update** function +2. **Implement brand store events** recording +3. **Test brand lifecycle** (add, remove, reappear scenarios) +4. **Verify event recording** accuracy + +### Phase 4: Aggregation Layer (Week 4) + +1. **Implement daily aggregation job** (`runDailyBrandAggregation`) +2. **Add cron schedule** (run at 2 AM daily) +3. **Test aggregation** with sample data +4. **Enable `BRAND_METRICS_ENABLED`** in staging + +### Phase 5: API Layer (Week 5) + +1. **Implement feature flag middleware** +2. **Implement brand authorization middleware** +3. **Implement all 6 brand intelligence endpoints** +4. **Add OpenAPI documentation** +5. **Enable `INTELLIGENCE_API_ENABLED`** in staging + +### Phase 6: Testing & QA (Week 6) + +1. **End-to-end testing** of full data flow +2. **Load testing** of API endpoints +3. **Verify rollback procedures** work correctly +4. **Performance profiling** and index tuning +5. **Security review** of brand scoping + +### Phase 7: Production Deployment (Week 7) + +1. **Deploy migration** to production +2. **Enable ingestion** (`INTELLIGENCE_INGEST_ENABLED = TRUE`) +3. **Monitor for 48 hours** +4. **Enable aggregation** (`BRAND_METRICS_ENABLED = TRUE`) +5. **Enable API** (`INTELLIGENCE_API_ENABLED = TRUE`) +6. **Announce to Cannabrands team** + +### Phase 8: Monitoring & Iteration (Ongoing) + +1. **Add Datadog/monitoring** for API latency and error rates +2. **Track aggregation job** duration and success +3. **Monitor table sizes** and plan partitioning if needed +4. **Gather feedback** from Cannabrands users +5. **Iterate on API** based on usage patterns + +--- + +## 7. Performance Considerations + +### 7.1 Indexing Strategy + +- All foreign keys are indexed +- Composite indexes on common query patterns +- Partial indexes for active-only queries +- GIN index on brand aliases array + +### 7.2 Partitioning (Future) + +If `store_product_snapshots` grows large: +```sql +-- Partition by month +ALTER TABLE store_product_snapshots + PARTITION BY RANGE (captured_at); +``` + +### 7.3 Query Optimization + +- Pre-aggregated tables avoid expensive JOINs on dashboards +- Daily aggregation runs during off-peak hours +- API endpoints use indexed columns for filtering + +### 7.4 Caching (Future) + +- Consider Redis caching for `/summary` endpoint +- Cache brand daily metrics with 1-hour TTL +- Invalidate on aggregation completion + +--- + +## 8. Security Considerations + +### 8.1 Multi-Tenant Isolation + +- All brand-scoped endpoints require JWT with `brand_key` claim +- Middleware enforces brand_key matches request parameter +- No cross-brand data leakage possible + +### 8.2 Rate Limiting + +- Apply standard rate limits to intelligence API +- Higher limits for authenticated brand users +- Monitor for abuse patterns + +### 8.3 Data Access + +- WordPress plugin continues using existing endpoints (unchanged) +- Brand users only see their own brand data +- Admin users can access all data via separate admin endpoints + +--- + +## Appendix A: Migration File Template + +```sql +-- migrations/YYYYMMDD_add_intelligence_layer.sql + +BEGIN; + +-- ENUMs +CREATE TYPE product_change_type AS ENUM ('NEW', 'CHANGED', 'UNCHANGED', 'STALE', 'REMOVED'); +CREATE TYPE special_type AS ENUM ('percent_off', 'dollar_off', 'bogo', 'bundle', 'set_price', 'other'); +CREATE TYPE brand_event_type AS ENUM ('ADDED', 'REMOVED', 'REAPPEARED'); + +-- Feature flags +CREATE TABLE IF NOT EXISTS feature_flags ( + id SERIAL PRIMARY KEY, + name VARCHAR(100) NOT NULL UNIQUE, + enabled BOOLEAN NOT NULL DEFAULT FALSE, + metadata JSONB DEFAULT '{}', + updated_at TIMESTAMPTZ DEFAULT NOW() +); + +INSERT INTO feature_flags (name, enabled) VALUES + ('INTELLIGENCE_ENABLED', FALSE), + ('INTELLIGENCE_INGEST_ENABLED', FALSE), + ('INTELLIGENCE_API_ENABLED', FALSE), + ('BRAND_METRICS_ENABLED', FALSE) +ON CONFLICT (name) DO NOTHING; + +-- [Include all CREATE TABLE statements from Section 1] + +COMMIT; +``` + +--- + +## Appendix B: Environment Variables + +```bash +# Feature flags can also be controlled via env vars (override DB) +INTELLIGENCE_ENABLED=false +INTELLIGENCE_INGEST_ENABLED=false +INTELLIGENCE_API_ENABLED=false +BRAND_METRICS_ENABLED=false + +# Aggregation job schedule +AGGREGATION_CRON="0 2 * * *" # 2 AM daily + +# Staleness thresholds +STALE_THRESHOLD_CRAWLS=3 +REMOVED_THRESHOLD_CRAWLS=7 +``` diff --git a/docs/PRODUCT_BRAND_INTELLIGENCE_FINAL.md b/docs/PRODUCT_BRAND_INTELLIGENCE_FINAL.md new file mode 100644 index 00000000..3bc70164 --- /dev/null +++ b/docs/PRODUCT_BRAND_INTELLIGENCE_FINAL.md @@ -0,0 +1,2027 @@ +# Product & Brand Intelligence Layer - Final Specification + +## Overview + +This document provides the **complete, explicit specification** for the Brand Intelligence Layer. All SQL is production-ready. No placeholders. + +### Non-Negotiable Constraints + +1. **DO NOT** modify or redesign the crawler +2. **DO NOT** break existing API endpoints +3. **DO NOT** break the WordPress plugin integration +4. Intelligence layer must be **additive only** +5. Rollback via **feature flags**, not schema deletion + +> **See Also**: [CRAWL_OPERATIONS.md](./CRAWL_OPERATIONS.md) for complete crawler policy, scheduling, and data integrity requirements. + +### Frozen Crawler Policy + +The crawler is **FROZEN**. This means: + +| Component | Status | Rationale | +|-----------|--------|-----------| +| CSS/XPath Selectors | FROZEN | Stability - tested against live Dutchie | +| Parsing Logic | FROZEN | Data integrity - validated extraction | +| Request Patterns | FROZEN | Anti-detection - proven configuration | +| Browser Configuration | FROZEN | Performance - optimized settings | + +**What CAN be modified:** +- Scheduling (when/how often crawls run) +- Post-crawl ingestion (how data flows into intelligence tables) +- API queries (how data is retrieved and transformed) +- Aggregation jobs (how metrics are computed) + +### Append-Only Data Philosophy + +> **Principle**: Every crawl should ADD information, never LOSE it. + +| Action | Allowed | Not Allowed | +|--------|---------|-------------| +| INSERT new product | Yes | - | +| UPDATE product fields | Yes | - | +| Mark product stale/removed | Yes | - | +| DELETE product row | No | Never delete | +| TRUNCATE table | No | Never truncate | +| UPDATE to NULL existing data | No | Never null-out | + +**Key Tables:** +- `store_product_snapshots` - Strictly INSERT-only (one row per product per crawl) +- `products` - UPDATE allowed, DELETE never +- `crawl_runs` - INSERT-only audit trail + +**Product Lifecycle States:** +``` +active → stale (3+ missed crawls) → removed (7+ missed crawls) + ↓ + archived (manual) +``` + +Products move through states but are **never deleted** from the database + +### Feature Flags Summary + +| Flag | Purpose | Default | +|------|---------|---------| +| `INTELLIGENCE_ENABLED` | Master kill switch | `FALSE` | +| `INTELLIGENCE_INGEST_ENABLED` | Controls crawl ingestion | `FALSE` | +| `INTELLIGENCE_API_ENABLED` | Controls API endpoints | `FALSE` | +| `BRAND_METRICS_ENABLED` | Controls daily aggregation | `FALSE` | + +**Normal rollback = flip flags, not drop tables.** + +--- + +## 1. ENUM Definitions + +```sql +-- Product change classification +CREATE TYPE product_change_type AS ENUM ( + 'new', -- First time seen at this store + 'changed', -- Price, stock, or special changed + 'unchanged', -- No changes detected + 'stale', -- Missing from 3+ consecutive crawls + 'removed' -- Missing from 7+ consecutive crawls +); + +-- Special/deal types +CREATE TYPE special_type AS ENUM ( + 'none', -- No special + 'percent_off', -- X% off + 'dollar_off', -- $X off + 'bogo', -- Buy one get one + 'bundle', -- X for $Y + 'set_price', -- Now $X + 'other' -- Unparseable special text +); + +-- Brand store event types +CREATE TYPE brand_event_type AS ENUM ( + 'added', -- Brand appeared at store + 'removed', -- Brand disappeared from store + 'reappeared' -- Brand returned after removal +); +``` + +--- + +## 2. Complete Table Definitions + +### 2.1 Feature Flags Table + +```sql +CREATE TABLE feature_flags ( + id SERIAL PRIMARY KEY, + name VARCHAR(100) NOT NULL, + enabled BOOLEAN NOT NULL DEFAULT FALSE, + description TEXT, + metadata JSONB DEFAULT '{}', + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + + CONSTRAINT uq_feature_flags_name UNIQUE (name) +); + +-- Initial flags +INSERT INTO feature_flags (name, enabled, description) VALUES + ('INTELLIGENCE_ENABLED', FALSE, 'Master kill switch for intelligence layer'), + ('INTELLIGENCE_INGEST_ENABLED', FALSE, 'Controls crawl result ingestion'), + ('INTELLIGENCE_API_ENABLED', FALSE, 'Controls brand intelligence API'), + ('BRAND_METRICS_ENABLED', FALSE, 'Controls daily aggregation job') +ON CONFLICT (name) DO NOTHING; + +CREATE INDEX idx_feature_flags_name ON feature_flags(name); +``` + +### 2.2 Crawl Runs Table + +```sql +CREATE TABLE crawl_runs ( + id SERIAL PRIMARY KEY, + store_id INTEGER NOT NULL, + dispensary_id INTEGER NOT NULL, + + -- Timing + started_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + completed_at TIMESTAMPTZ, + + -- Status + status VARCHAR(20) NOT NULL DEFAULT 'running', + -- Values: 'running', 'completed', 'failed', 'cancelled' + + -- Metrics + products_found INTEGER DEFAULT 0, + products_new INTEGER DEFAULT 0, + products_changed INTEGER DEFAULT 0, + products_unchanged INTEGER DEFAULT 0, + products_stale INTEGER DEFAULT 0, + + -- Error handling + error_message TEXT, + error_stack TEXT, + + -- Metadata + metadata JSONB DEFAULT '{}', + + -- Foreign keys (soft - stores/dispensaries may not exist in all envs) + -- REFERENCES stores(id) ON DELETE CASCADE, + -- REFERENCES dispensaries(id) ON DELETE CASCADE + + CONSTRAINT chk_crawl_status CHECK (status IN ('running', 'completed', 'failed', 'cancelled')) +); + +CREATE INDEX idx_crawl_runs_store_id ON crawl_runs(store_id); +CREATE INDEX idx_crawl_runs_dispensary_id ON crawl_runs(dispensary_id); +CREATE INDEX idx_crawl_runs_started_at ON crawl_runs(started_at DESC); +CREATE INDEX idx_crawl_runs_store_started ON crawl_runs(store_id, started_at DESC); +CREATE INDEX idx_crawl_runs_status ON crawl_runs(status) WHERE status = 'running'; +``` + +### 2.3 Canonical Brands Table (New Columns Only) + +```sql +-- Assuming canonical_brands table already exists, add these columns: +ALTER TABLE canonical_brands + ADD COLUMN IF NOT EXISTS cannabrands_id VARCHAR(100), + ADD COLUMN IF NOT EXISTS cannabrands_slug VARCHAR(255), + ADD COLUMN IF NOT EXISTS is_verified BOOLEAN NOT NULL DEFAULT FALSE, + ADD COLUMN IF NOT EXISTS aliases TEXT[] DEFAULT '{}'; + +-- Add unique constraint on cannabrands_id +ALTER TABLE canonical_brands + ADD CONSTRAINT uq_canonical_brands_cannabrands_id UNIQUE (cannabrands_id); + +-- Indexes +CREATE INDEX IF NOT EXISTS idx_canonical_brands_cannabrands_id + ON canonical_brands(cannabrands_id) WHERE cannabrands_id IS NOT NULL; + +CREATE INDEX IF NOT EXISTS idx_canonical_brands_cannabrands_slug + ON canonical_brands(cannabrands_slug) WHERE cannabrands_slug IS NOT NULL; + +CREATE INDEX IF NOT EXISTS idx_canonical_brands_aliases + ON canonical_brands USING GIN(aliases); + +CREATE INDEX IF NOT EXISTS idx_canonical_brands_name_lower + ON canonical_brands(LOWER(name)); +``` + +### 2.4 Brand Aliases Table (Optional - For Complex Alias Mapping) + +```sql +CREATE TABLE brand_aliases ( + id SERIAL PRIMARY KEY, + canonical_brand_id INTEGER NOT NULL, + alias VARCHAR(255) NOT NULL, + source VARCHAR(50) DEFAULT 'manual', + -- Values: 'manual', 'auto', 'dutchie', 'leafly' + confidence NUMERIC(3,2) DEFAULT 1.00, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + + -- Foreign key + CONSTRAINT fk_brand_aliases_brand FOREIGN KEY (canonical_brand_id) + REFERENCES canonical_brands(id) ON DELETE CASCADE, + + -- Unique alias per brand + CONSTRAINT uq_brand_aliases_alias UNIQUE (alias) +); + +CREATE INDEX idx_brand_aliases_brand_id ON brand_aliases(canonical_brand_id); +CREATE INDEX idx_brand_aliases_alias_lower ON brand_aliases(LOWER(alias)); +``` + +### 2.5 Store Products Table + +```sql +CREATE TABLE store_products ( + id SERIAL PRIMARY KEY, + store_id INTEGER NOT NULL, + dispensary_id INTEGER NOT NULL, + product_id INTEGER NOT NULL, + canonical_brand_id INTEGER, + + -- Fingerprint for deduplication (SHA256 of normalized name+brand+variant+weight) + fingerprint VARCHAR(64) NOT NULL, + + -- Lifecycle tracking + first_seen_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + last_seen_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + last_changed_at TIMESTAMPTZ, + removed_at TIMESTAMPTZ, + + -- Current state + current_price NUMERIC(10,2), + current_in_stock BOOLEAN DEFAULT TRUE, + current_special_text TEXT, + current_special_type special_type DEFAULT 'none', + current_special_value NUMERIC(10,2), + current_special_data JSONB, -- Parsed special details + + -- Staleness tracking + consecutive_missing INTEGER NOT NULL DEFAULT 0, + change_status product_change_type NOT NULL DEFAULT 'new', + + -- Metadata + metadata JSONB DEFAULT '{}', + + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + + -- Constraints + CONSTRAINT uq_store_products_fingerprint UNIQUE (store_id, fingerprint), + CONSTRAINT fk_store_products_brand FOREIGN KEY (canonical_brand_id) + REFERENCES canonical_brands(id) ON DELETE SET NULL +); + +-- Primary lookup indexes +CREATE INDEX idx_store_products_store_id ON store_products(store_id); +CREATE INDEX idx_store_products_dispensary_id ON store_products(dispensary_id); +CREATE INDEX idx_store_products_product_id ON store_products(product_id); +CREATE INDEX idx_store_products_brand_id ON store_products(canonical_brand_id); +CREATE INDEX idx_store_products_fingerprint ON store_products(fingerprint); + +-- Status/filtering indexes +CREATE INDEX idx_store_products_status ON store_products(change_status); +CREATE INDEX idx_store_products_store_status ON store_products(store_id, change_status); + +-- Active products only (partial index) +CREATE INDEX idx_store_products_active ON store_products(store_id, canonical_brand_id) + WHERE change_status NOT IN ('stale', 'removed'); + +-- Brand + store combo for presence calculations +CREATE INDEX idx_store_products_brand_store ON store_products(canonical_brand_id, store_id); + +-- Time-based queries +CREATE INDEX idx_store_products_last_seen ON store_products(last_seen_at DESC); +CREATE INDEX idx_store_products_first_seen ON store_products(first_seen_at DESC); +``` + +### 2.6 Store Product Snapshots Table + +```sql +CREATE TABLE store_product_snapshots ( + id BIGSERIAL PRIMARY KEY, + store_product_id INTEGER NOT NULL, + crawl_run_id INTEGER NOT NULL, + + -- Point-in-time state + price NUMERIC(10,2), + in_stock BOOLEAN, + special_text TEXT, + special_type special_type DEFAULT 'none', + special_value NUMERIC(10,2), + special_data JSONB, -- Parsed special details + + -- Change detection + change_type product_change_type NOT NULL, + changes_detected JSONB, -- {"price": {"old": 25.00, "new": 30.00}, ...} + + -- Timestamp + captured_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + + -- Foreign keys + CONSTRAINT fk_snapshots_store_product FOREIGN KEY (store_product_id) + REFERENCES store_products(id) ON DELETE CASCADE, + CONSTRAINT fk_snapshots_crawl_run FOREIGN KEY (crawl_run_id) + REFERENCES crawl_runs(id) ON DELETE CASCADE, + + -- Prevent duplicate snapshots for same product in same crawl + CONSTRAINT uq_snapshots_product_crawl UNIQUE (store_product_id, crawl_run_id) +); + +-- Primary lookup indexes +CREATE INDEX idx_snapshots_store_product_id ON store_product_snapshots(store_product_id); +CREATE INDEX idx_snapshots_crawl_run_id ON store_product_snapshots(crawl_run_id); + +-- Time-series queries +CREATE INDEX idx_snapshots_product_time ON store_product_snapshots(store_product_id, captured_at DESC); + +-- Change type filtering +CREATE INDEX idx_snapshots_change_type ON store_product_snapshots(change_type); +CREATE INDEX idx_snapshots_crawl_change ON store_product_snapshots(crawl_run_id, change_type); +``` + +### 2.7 Brand Store Presence Table + +```sql +CREATE TABLE brand_store_presence ( + id SERIAL PRIMARY KEY, + canonical_brand_id INTEGER NOT NULL, + store_id INTEGER NOT NULL, + dispensary_id INTEGER NOT NULL, + + -- Presence tracking + first_seen_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + last_seen_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + removed_at TIMESTAMPTZ, + + -- Current metrics + active_sku_count INTEGER NOT NULL DEFAULT 0, + in_stock_sku_count INTEGER NOT NULL DEFAULT 0, + total_sku_count INTEGER NOT NULL DEFAULT 0, -- Including removed + + -- Status + is_active BOOLEAN NOT NULL DEFAULT TRUE, + consecutive_missing INTEGER NOT NULL DEFAULT 0, + + -- Timestamps + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + + -- Foreign keys + CONSTRAINT fk_brand_presence_brand FOREIGN KEY (canonical_brand_id) + REFERENCES canonical_brands(id) ON DELETE CASCADE, + + -- Unique brand+store combo + CONSTRAINT uq_brand_presence_brand_store UNIQUE (canonical_brand_id, store_id) +); + +-- Primary lookup indexes +CREATE INDEX idx_brand_presence_brand_id ON brand_store_presence(canonical_brand_id); +CREATE INDEX idx_brand_presence_store_id ON brand_store_presence(store_id); +CREATE INDEX idx_brand_presence_dispensary_id ON brand_store_presence(dispensary_id); + +-- Brand + store combo +CREATE INDEX idx_brand_presence_brand_store ON brand_store_presence(canonical_brand_id, store_id); + +-- Active only (partial index) +CREATE INDEX idx_brand_presence_active ON brand_store_presence(canonical_brand_id) + WHERE is_active = TRUE; + +-- Store + active for dashboard queries +CREATE INDEX idx_brand_presence_store_active ON brand_store_presence(store_id, is_active); +``` + +### 2.8 Brand Daily Metrics Table + +```sql +CREATE TABLE brand_daily_metrics ( + id BIGSERIAL PRIMARY KEY, + canonical_brand_id INTEGER NOT NULL, + metric_date DATE NOT NULL, + + -- Store counts + total_stores INTEGER NOT NULL DEFAULT 0, + stores_added INTEGER NOT NULL DEFAULT 0, + stores_removed INTEGER NOT NULL DEFAULT 0, + + -- SKU counts + total_skus INTEGER NOT NULL DEFAULT 0, + skus_in_stock INTEGER NOT NULL DEFAULT 0, + new_skus INTEGER NOT NULL DEFAULT 0, + removed_skus INTEGER NOT NULL DEFAULT 0, + + -- Price metrics (in cents for precision) + avg_price_cents INTEGER, + min_price_cents INTEGER, + max_price_cents INTEGER, + price_changes INTEGER NOT NULL DEFAULT 0, + + -- Promo summary + skus_on_promo INTEGER NOT NULL DEFAULT 0, + + -- Computation metadata + computed_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + computation_ms INTEGER, -- How long aggregation took + + -- Foreign key + CONSTRAINT fk_brand_metrics_brand FOREIGN KEY (canonical_brand_id) + REFERENCES canonical_brands(id) ON DELETE CASCADE, + + -- One row per brand per date + CONSTRAINT uq_brand_metrics_brand_date UNIQUE (canonical_brand_id, metric_date) +); + +-- Primary lookup +CREATE INDEX idx_brand_metrics_brand_id ON brand_daily_metrics(canonical_brand_id); +CREATE INDEX idx_brand_metrics_date ON brand_daily_metrics(metric_date DESC); + +-- Brand + date combo (most common query pattern) +CREATE INDEX idx_brand_metrics_brand_date ON brand_daily_metrics(canonical_brand_id, metric_date DESC); + +-- Date range queries +CREATE INDEX idx_brand_metrics_date_brand ON brand_daily_metrics(metric_date, canonical_brand_id); +``` + +### 2.9 Brand Store Events Table + +```sql +CREATE TABLE brand_store_events ( + id BIGSERIAL PRIMARY KEY, + canonical_brand_id INTEGER NOT NULL, + store_id INTEGER NOT NULL, + dispensary_id INTEGER NOT NULL, + + -- Event details + event_type brand_event_type NOT NULL, + event_date DATE NOT NULL, + + -- Optional crawl reference + crawl_run_id INTEGER, + + -- Metrics at time of event + sku_count INTEGER, + + -- Notes/metadata + notes TEXT, + metadata JSONB DEFAULT '{}', + + -- Timestamps + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + + -- Foreign keys + CONSTRAINT fk_brand_events_brand FOREIGN KEY (canonical_brand_id) + REFERENCES canonical_brands(id) ON DELETE CASCADE, + CONSTRAINT fk_brand_events_crawl FOREIGN KEY (crawl_run_id) + REFERENCES crawl_runs(id) ON DELETE SET NULL, + + -- Prevent duplicate events for same brand+store+date+type + CONSTRAINT uq_brand_events_unique UNIQUE (canonical_brand_id, store_id, event_date, event_type) +); + +-- Primary lookups +CREATE INDEX idx_brand_events_brand_id ON brand_store_events(canonical_brand_id); +CREATE INDEX idx_brand_events_store_id ON brand_store_events(store_id); + +-- Timeline queries +CREATE INDEX idx_brand_events_brand_date ON brand_store_events(canonical_brand_id, event_date DESC); +CREATE INDEX idx_brand_events_store_date ON brand_store_events(store_id, event_date DESC); + +-- Event type filtering +CREATE INDEX idx_brand_events_type ON brand_store_events(event_type); +CREATE INDEX idx_brand_events_brand_type ON brand_store_events(canonical_brand_id, event_type, event_date DESC); + +-- Crawl reference +CREATE INDEX idx_brand_events_crawl ON brand_store_events(crawl_run_id) + WHERE crawl_run_id IS NOT NULL; +``` + +### 2.10 Brand Promo Daily Metrics Table + +```sql +CREATE TABLE brand_promo_daily_metrics ( + id BIGSERIAL PRIMARY KEY, + canonical_brand_id INTEGER NOT NULL, + metric_date DATE NOT NULL, + + -- Total promos + total_promos INTEGER NOT NULL DEFAULT 0, + + -- Breakdown by type + percent_off_count INTEGER NOT NULL DEFAULT 0, + dollar_off_count INTEGER NOT NULL DEFAULT 0, + bogo_count INTEGER NOT NULL DEFAULT 0, + bundle_count INTEGER NOT NULL DEFAULT 0, + set_price_count INTEGER NOT NULL DEFAULT 0, + other_count INTEGER NOT NULL DEFAULT 0, + + -- Value metrics + avg_discount_percent NUMERIC(5,2), + total_discount_value NUMERIC(12,2), + + -- Store breakdown + stores_with_promos INTEGER NOT NULL DEFAULT 0, + + -- Computation metadata + computed_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + + -- Foreign key + CONSTRAINT fk_brand_promo_metrics_brand FOREIGN KEY (canonical_brand_id) + REFERENCES canonical_brands(id) ON DELETE CASCADE, + + -- One row per brand per date + CONSTRAINT uq_brand_promo_metrics_brand_date UNIQUE (canonical_brand_id, metric_date) +); + +-- Primary lookups +CREATE INDEX idx_brand_promo_metrics_brand_id ON brand_promo_daily_metrics(canonical_brand_id); +CREATE INDEX idx_brand_promo_metrics_date ON brand_promo_daily_metrics(metric_date DESC); + +-- Brand + date combo +CREATE INDEX idx_brand_promo_metrics_brand_date ON brand_promo_daily_metrics(canonical_brand_id, metric_date DESC); +``` + +### 2.11 Brand Store Detail View + +```sql +CREATE OR REPLACE VIEW v_brand_store_detail AS +SELECT + cb.id AS brand_id, + cb.name AS brand_name, + cb.cannabrands_id AS brand_key, + cb.cannabrands_slug AS brand_slug, + s.id AS store_id, + s.name AS store_name, + s.slug AS store_slug, + d.id AS dispensary_id, + d.name AS dispensary_name, + d.city AS city, + d.state AS state, + d.dutchie_url AS dutchie_url, + bsp.first_seen_at AS first_seen_at, + bsp.last_seen_at AS last_seen_at, + bsp.removed_at AS removed_at, + bsp.is_active AS is_active, + bsp.active_sku_count AS active_sku_count, + bsp.in_stock_sku_count AS in_stock_sku_count, + bsp.total_sku_count AS total_sku_count, + EXTRACT(DAY FROM NOW() - bsp.last_seen_at)::INTEGER AS days_since_seen, + EXTRACT(DAY FROM COALESCE(bsp.removed_at, NOW()) - bsp.first_seen_at)::INTEGER AS tenure_days +FROM brand_store_presence bsp +JOIN canonical_brands cb ON cb.id = bsp.canonical_brand_id +JOIN stores s ON s.id = bsp.store_id +JOIN dispensaries d ON d.id = bsp.dispensary_id; + +COMMENT ON VIEW v_brand_store_detail IS 'Per-store brand detail with aggregated metrics'; +``` + +--- + +## 3. Special Text Parsing Format + +### 3.1 Parsed Special Data Structure + +The `current_special_data` and `special_data` JSONB columns store parsed special details: + +```typescript +interface ParsedSpecial { + type: 'none' | 'percent_off' | 'dollar_off' | 'bogo' | 'bundle' | 'set_price' | 'other'; + raw_text: string; // Original special text + + // For percent_off + percent?: number; // e.g., 20 for "20% off" + + // For dollar_off + dollars?: number; // e.g., 5 for "$5 off" + + // For bogo + buy?: number; // e.g., 2 for "Buy 2 get 1" + get?: number; // e.g., 1 + effective_discount?: number; // Calculated % discount + + // For bundle + quantity?: number; // e.g., 3 for "3 for $100" + bundle_price?: number; // e.g., 100 + + // For set_price + set_price?: number; // e.g., 25 for "Now $25" +} +``` + +### 3.2 Example Parsed Values + +```json +// "20% off" +{ + "type": "percent_off", + "raw_text": "20% off", + "percent": 20 +} + +// "$5 off" +{ + "type": "dollar_off", + "raw_text": "$5 off", + "dollars": 5 +} + +// "BOGO" or "Buy One Get One Free" +{ + "type": "bogo", + "raw_text": "Buy One Get One Free", + "buy": 1, + "get": 1, + "effective_discount": 50 +} + +// "Buy 2 Get 1 Free" +{ + "type": "bogo", + "raw_text": "Buy 2 Get 1 Free", + "buy": 2, + "get": 1, + "effective_discount": 33.33 +} + +// "3 for $100" +{ + "type": "bundle", + "raw_text": "3 for $100", + "quantity": 3, + "bundle_price": 100 +} + +// "Now $25" +{ + "type": "set_price", + "raw_text": "Now $25", + "set_price": 25 +} + +// Unparseable: "Daily Special!" +{ + "type": "other", + "raw_text": "Daily Special!" +} +``` + +### 3.3 Parsing Implementation + +```typescript +interface ParsedSpecial { + type: 'none' | 'percent_off' | 'dollar_off' | 'bogo' | 'bundle' | 'set_price' | 'other'; + raw_text: string; + value: number | null; // Primary numeric value for DB column + data: Record; // Full parsed data for JSONB +} + +function parseSpecialText(text: string | null | undefined): ParsedSpecial { + if (!text || text.trim() === '') { + return { type: 'none', raw_text: '', value: null, data: {} }; + } + + const rawText = text.trim(); + const lower = rawText.toLowerCase(); + + // 1. Percent off: "20% off", "25% OFF", "Save 30%" + const pctMatch = lower.match(/(\d+(?:\.\d+)?)\s*%\s*(?:off|discount)?/i) + || lower.match(/save\s*(\d+(?:\.\d+)?)\s*%/i); + if (pctMatch) { + const percent = parseFloat(pctMatch[1]); + return { + type: 'percent_off', + raw_text: rawText, + value: percent, + data: { percent } + }; + } + + // 2. Dollar off: "$5 off", "$10 OFF" + const dollarMatch = lower.match(/\$(\d+(?:\.\d+)?)\s*off/i); + if (dollarMatch) { + const dollars = parseFloat(dollarMatch[1]); + return { + type: 'dollar_off', + raw_text: rawText, + value: dollars, + data: { dollars } + }; + } + + // 3. BOGO variants + // "BOGO", "Buy One Get One", "B1G1", "Buy 2 Get 1" + const bogoSimple = /\bbogo\b|buy\s*one\s*get\s*one/i.test(lower); + const bogoMatch = lower.match(/buy\s*(\d+)\s*get\s*(\d+)/i); + + if (bogoSimple) { + return { + type: 'bogo', + raw_text: rawText, + value: 50, // 50% effective discount + data: { buy: 1, get: 1, effective_discount: 50 } + }; + } + + if (bogoMatch) { + const buy = parseInt(bogoMatch[1]); + const get = parseInt(bogoMatch[2]); + const effectiveDiscount = (get / (buy + get)) * 100; + return { + type: 'bogo', + raw_text: rawText, + value: effectiveDiscount, + data: { buy, get, effective_discount: Math.round(effectiveDiscount * 100) / 100 } + }; + } + + // 4. Bundle: "3 for $100", "2/$50" + const bundleMatch = lower.match(/(\d+)\s*(?:for|\/)\s*\$(\d+(?:\.\d+)?)/i); + if (bundleMatch) { + const quantity = parseInt(bundleMatch[1]); + const bundlePrice = parseFloat(bundleMatch[2]); + return { + type: 'bundle', + raw_text: rawText, + value: bundlePrice, + data: { quantity, bundle_price: bundlePrice } + }; + } + + // 5. Set price: "Now $25", "Only $30", "Just $20" + const setPriceMatch = lower.match(/(?:now|only|just|sale)\s*\$(\d+(?:\.\d+)?)/i); + if (setPriceMatch) { + const setPrice = parseFloat(setPriceMatch[1]); + return { + type: 'set_price', + raw_text: rawText, + value: setPrice, + data: { set_price: setPrice } + }; + } + + // 6. Other - has text but couldn't parse + return { + type: 'other', + raw_text: rawText, + value: null, + data: {} + }; +} +``` + +--- + +## 4. Idempotency Requirements + +### 4.1 Ingestion Idempotency + +**Problem:** Same crawl might be processed twice (retry, duplicate event, etc.) + +**Solution:** Use `UNIQUE` constraint on `(store_product_id, crawl_run_id)` in snapshots table plus `ON CONFLICT` handling. + +```typescript +async function ingestCrawlResults( + db: Pool, + storeId: number, + dispensaryId: number, + crawledProducts: CrawledProduct[], + existingCrawlRunId?: number // Optional: resume existing crawl +): Promise { + const client = await db.connect(); + + try { + await client.query('BEGIN'); + + // Check feature flag + const flagEnabled = await isIngestEnabled(client); + if (!flagEnabled) { + await client.query('ROLLBACK'); + return createEmptyResult(); + } + + // Create or reuse crawl run + let crawlRunId: number; + + if (existingCrawlRunId) { + // Resume existing crawl - check it's still running + const existing = await client.query( + `SELECT id FROM crawl_runs WHERE id = $1 AND status = 'running'`, + [existingCrawlRunId] + ); + if (existing.rows.length === 0) { + throw new Error(`Crawl run ${existingCrawlRunId} not found or not running`); + } + crawlRunId = existingCrawlRunId; + } else { + // Check for existing running crawl for this store (prevent duplicates) + const runningCrawl = await client.query( + `SELECT id FROM crawl_runs + WHERE store_id = $1 AND status = 'running' + AND started_at > NOW() - INTERVAL '1 hour' + LIMIT 1`, + [storeId] + ); + + if (runningCrawl.rows.length > 0) { + // Reuse existing running crawl + crawlRunId = runningCrawl.rows[0].id; + } else { + // Create new crawl run + const newCrawl = await client.query( + `INSERT INTO crawl_runs (store_id, dispensary_id, status) + VALUES ($1, $2, 'running') + RETURNING id`, + [storeId, dispensaryId] + ); + crawlRunId = newCrawl.rows[0].id; + } + } + + // Process products... + // (main ingestion logic) + + // IDEMPOTENT SNAPSHOT INSERT + // Uses ON CONFLICT to handle duplicate (store_product_id, crawl_run_id) + await client.query(` + INSERT INTO store_product_snapshots + (store_product_id, crawl_run_id, price, in_stock, special_text, + special_type, special_value, special_data, change_type, changes_detected) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) + ON CONFLICT (store_product_id, crawl_run_id) DO UPDATE SET + price = EXCLUDED.price, + in_stock = EXCLUDED.in_stock, + special_text = EXCLUDED.special_text, + special_type = EXCLUDED.special_type, + special_value = EXCLUDED.special_value, + special_data = EXCLUDED.special_data, + change_type = EXCLUDED.change_type, + changes_detected = EXCLUDED.changes_detected, + captured_at = NOW() + `, [storeProductId, crawlRunId, price, inStock, specialText, + specialType, specialValue, specialData, changeType, changesDetected]); + + // IDEMPOTENT BRAND EVENT INSERT + // Uses ON CONFLICT to handle duplicate (brand_id, store_id, date, type) + await client.query(` + INSERT INTO brand_store_events + (canonical_brand_id, store_id, dispensary_id, event_type, event_date, crawl_run_id, sku_count) + VALUES ($1, $2, $3, $4, CURRENT_DATE, $5, $6) + ON CONFLICT (canonical_brand_id, store_id, event_date, event_type) DO UPDATE SET + crawl_run_id = EXCLUDED.crawl_run_id, + sku_count = EXCLUDED.sku_count, + created_at = NOW() + `, [brandId, storeId, dispensaryId, eventType, crawlRunId, skuCount]); + + await client.query('COMMIT'); + return result; + + } catch (error) { + await client.query('ROLLBACK'); + throw error; + } finally { + client.release(); + } +} +``` + +### 4.2 Daily Aggregation Idempotency + +**Problem:** Aggregation job might run twice for the same date (cron overlap, manual trigger, etc.) + +**Solution:** Use `INSERT ... ON CONFLICT ... DO UPDATE` pattern. + +```typescript +async function aggregateBrandDailyMetrics( + db: Pool, + brandId: number, + targetDate: string // YYYY-MM-DD +): Promise { + const client = await db.connect(); + + try { + await client.query('BEGIN'); + + const startTime = Date.now(); + + // Compute metrics + const metrics = await client.query(` + SELECT + COUNT(DISTINCT bsp.store_id) FILTER (WHERE bsp.is_active) as total_stores, + COUNT(DISTINCT bse.store_id) FILTER ( + WHERE bse.event_type = 'added' AND bse.event_date = $2::date + ) as stores_added, + COUNT(DISTINCT bse.store_id) FILTER ( + WHERE bse.event_type = 'removed' AND bse.event_date = $2::date + ) as stores_removed, + COUNT(sp.id) FILTER ( + WHERE sp.change_status NOT IN ('stale', 'removed') + ) as total_skus, + COUNT(sp.id) FILTER ( + WHERE sp.change_status NOT IN ('stale', 'removed') AND sp.current_in_stock + ) as skus_in_stock, + COUNT(sp.id) FILTER ( + WHERE sp.change_status = 'new' AND sp.first_seen_at::date = $2::date + ) as new_skus, + COUNT(sp.id) FILTER ( + WHERE sp.change_status = 'removed' AND sp.removed_at::date = $2::date + ) as removed_skus, + ROUND(AVG(sp.current_price) FILTER ( + WHERE sp.change_status NOT IN ('stale', 'removed') AND sp.current_price IS NOT NULL + ) * 100)::integer as avg_price_cents, + ROUND(MIN(sp.current_price) FILTER ( + WHERE sp.change_status NOT IN ('stale', 'removed') AND sp.current_price IS NOT NULL + ) * 100)::integer as min_price_cents, + ROUND(MAX(sp.current_price) FILTER ( + WHERE sp.change_status NOT IN ('stale', 'removed') AND sp.current_price IS NOT NULL + ) * 100)::integer as max_price_cents, + COUNT(sp.id) FILTER ( + WHERE sp.current_special_type != 'none' AND sp.change_status NOT IN ('stale', 'removed') + ) as skus_on_promo + FROM canonical_brands cb + LEFT JOIN store_products sp ON sp.canonical_brand_id = cb.id + LEFT JOIN brand_store_presence bsp ON bsp.canonical_brand_id = cb.id + LEFT JOIN brand_store_events bse ON bse.canonical_brand_id = cb.id + WHERE cb.id = $1 + GROUP BY cb.id + `, [brandId, targetDate]); + + const m = metrics.rows[0] || {}; + const computationMs = Date.now() - startTime; + + // IDEMPOTENT UPSERT + await client.query(` + INSERT INTO brand_daily_metrics ( + canonical_brand_id, metric_date, + total_stores, stores_added, stores_removed, + total_skus, skus_in_stock, new_skus, removed_skus, + avg_price_cents, min_price_cents, max_price_cents, + price_changes, skus_on_promo, computed_at, computation_ms + ) VALUES ( + $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, NOW(), $15 + ) + ON CONFLICT (canonical_brand_id, metric_date) DO UPDATE SET + total_stores = EXCLUDED.total_stores, + stores_added = EXCLUDED.stores_added, + stores_removed = EXCLUDED.stores_removed, + total_skus = EXCLUDED.total_skus, + skus_in_stock = EXCLUDED.skus_in_stock, + new_skus = EXCLUDED.new_skus, + removed_skus = EXCLUDED.removed_skus, + avg_price_cents = EXCLUDED.avg_price_cents, + min_price_cents = EXCLUDED.min_price_cents, + max_price_cents = EXCLUDED.max_price_cents, + price_changes = EXCLUDED.price_changes, + skus_on_promo = EXCLUDED.skus_on_promo, + computed_at = NOW(), + computation_ms = EXCLUDED.computation_ms + `, [ + brandId, targetDate, + m.total_stores || 0, m.stores_added || 0, m.stores_removed || 0, + m.total_skus || 0, m.skus_in_stock || 0, m.new_skus || 0, m.removed_skus || 0, + m.avg_price_cents, m.min_price_cents, m.max_price_cents, + 0, m.skus_on_promo || 0, computationMs + ]); + + // Same pattern for promo metrics... + + await client.query('COMMIT'); + + } catch (error) { + await client.query('ROLLBACK'); + throw error; + } finally { + client.release(); + } +} +``` + +### 4.3 Backfill Idempotency + +**Problem:** Historical backfill might be run multiple times (interrupted, re-run with different date range, etc.) + +**Solution:** Same `ON CONFLICT DO UPDATE` pattern. Safe to re-run any date range. + +```typescript +async function backfillBrandMetrics( + db: Pool, + startDate: string, + endDate: string, + brandIds?: number[] +): Promise<{ daysProcessed: number; brandsProcessed: number }> { + // Generate date range + const dates = generateDateRange(startDate, endDate); + + // Get brands to process + const brandsQuery = brandIds?.length + ? `SELECT id FROM canonical_brands WHERE cannabrands_id IS NOT NULL AND id = ANY($1)` + : `SELECT id FROM canonical_brands WHERE cannabrands_id IS NOT NULL`; + + const brands = await db.query(brandsQuery, brandIds?.length ? [brandIds] : []); + + let daysProcessed = 0; + + for (const targetDate of dates) { + for (const brand of brands.rows) { + // Each call is idempotent via ON CONFLICT DO UPDATE + await aggregateBrandDailyMetrics(db, brand.id, targetDate); + await aggregateBrandPromoMetrics(db, brand.id, targetDate); + } + daysProcessed++; + } + + return { daysProcessed, brandsProcessed: brands.rows.length }; +} +``` + +--- + +## 5. Brand Key Mapping & Resolution + +### 5.1 Brand Key Definition + +The `brand_key` parameter in API endpoints maps to `canonical_brands.cannabrands_id`. + +| Field | Purpose | Example | +|-------|---------|---------| +| `cannabrands_id` | Primary API identifier, stable | `cb_12345` | +| `cannabrands_slug` | Human-readable, for URLs | `raw-garden` | + +### 5.2 Resolution Logic + +```typescript +/** + * Resolve brand_key to canonical_brand_id + * Tries cannabrands_id first, then cannabrands_slug as fallback + */ +async function resolveBrandKey(db: Pool, brandKey: string): Promise { + // Try cannabrands_id first (primary) + const byId = await db.query( + `SELECT id FROM canonical_brands WHERE cannabrands_id = $1`, + [brandKey] + ); + + if (byId.rows.length > 0) { + return byId.rows[0].id; + } + + // Fallback to cannabrands_slug + const bySlug = await db.query( + `SELECT id FROM canonical_brands WHERE cannabrands_slug = $1`, + [brandKey] + ); + + if (bySlug.rows.length > 0) { + return bySlug.rows[0].id; + } + + return null; +} +``` + +### 5.3 Brand Authorization Middleware + +```typescript +interface AuthenticatedRequest extends Request { + user: { + brand_key: string; // From JWT + permissions: string[]; + }; + canonicalBrandId: number; // Resolved brand ID +} + +async function requireBrandAccess( + req: AuthenticatedRequest, + res: Response, + next: NextFunction +) { + const { brand_key } = req.params; + const auth = req.user; + + // Verify JWT brand_key matches request brand_key + if (!auth || auth.brand_key !== brand_key) { + return res.status(403).json({ + error: 'Access denied', + code: 'BRAND_ACCESS_DENIED', + message: `You do not have access to brand: ${brand_key}` + }); + } + + // Resolve brand_key to canonical_brand_id + const brandId = await resolveBrandKey(db, brand_key); + + if (!brandId) { + return res.status(404).json({ + error: 'Brand not found', + code: 'BRAND_NOT_FOUND', + message: `No brand found with key: ${brand_key}` + }); + } + + // Attach resolved brand ID to request + req.canonicalBrandId = brandId; + + next(); +} +``` + +### 5.4 Resolution Flow Diagram + +``` +API Request: GET /api/brands/raw-garden/intelligence/summary + ↓ + brand_key = "raw-garden" + ↓ + ┌───────────────┴───────────────┐ + │ JWT Validation │ + │ req.user.brand_key must │ + │ match "raw-garden" │ + └───────────────┬───────────────┘ + ↓ + ┌───────────────┴───────────────┐ + │ Brand Resolution │ + │ 1. Try: cannabrands_id = │ + │ "raw-garden" │ + │ 2. Try: cannabrands_slug = │ + │ "raw-garden" │ + └───────────────┬───────────────┘ + ↓ + canonical_brand_id = 42 + ↓ + ┌───────────────┴───────────────┐ + │ All queries scoped: │ + │ WHERE canonical_brand_id = 42│ + └───────────────────────────────┘ +``` + +--- + +## 6. Feature Flag Implementation + +### 6.1 Feature Flag Service + +```typescript +// services/feature-flags.ts + +interface FeatureFlag { + name: string; + enabled: boolean; + metadata: Record; +} + +class FeatureFlagService { + private cache: Map = new Map(); + private readonly CACHE_TTL_MS = 60_000; // 60 seconds + + constructor(private db: Pool) {} + + async isEnabled(flagName: string): Promise { + // 1. Check environment variable override (highest priority) + const envValue = process.env[flagName]; + if (envValue !== undefined) { + return envValue.toLowerCase() === 'true'; + } + + // 2. Check cache + const cached = this.cache.get(flagName); + if (cached && cached.expires > Date.now()) { + return cached.value; + } + + // 3. Query database + try { + const result = await this.db.query( + `SELECT enabled FROM feature_flags WHERE name = $1`, + [flagName] + ); + + const enabled = result.rows[0]?.enabled ?? false; + + // Cache result + this.cache.set(flagName, { + value: enabled, + expires: Date.now() + this.CACHE_TTL_MS + }); + + return enabled; + } catch (error) { + console.error(`Failed to fetch feature flag ${flagName}:`, error); + return false; // Fail closed + } + } + + async setEnabled(flagName: string, enabled: boolean): Promise { + await this.db.query( + `UPDATE feature_flags SET enabled = $1, updated_at = NOW() WHERE name = $2`, + [enabled, flagName] + ); + + // Invalidate cache + this.cache.delete(flagName); + } + + clearCache(): void { + this.cache.clear(); + } +} + +// Singleton instance +export const featureFlags = new FeatureFlagService(db); +``` + +### 6.2 Flag Usage Patterns + +```typescript +// API Gate +async function requireIntelligenceApiEnabled(req: Request, res: Response, next: NextFunction) { + const enabled = await featureFlags.isEnabled('INTELLIGENCE_API_ENABLED'); + + if (!enabled) { + return res.status(404).json({ + error: 'Feature not available', + code: 'FEATURE_DISABLED' + }); + } + + next(); +} + +// Ingestion Gate +async function processCompletedCrawl(storeId: number, products: Product[]) { + const enabled = await featureFlags.isEnabled('INTELLIGENCE_INGEST_ENABLED'); + + if (!enabled) { + console.log('Intelligence ingestion disabled, skipping'); + return; + } + + await ingestCrawlResults(db, storeId, products); +} + +// Aggregation Gate +async function runDailyAggregationJob() { + const enabled = await featureFlags.isEnabled('BRAND_METRICS_ENABLED'); + + if (!enabled) { + console.log('Brand metrics aggregation disabled, skipping'); + return; + } + + await runDailyBrandAggregation(db); +} + +// Router setup +const intelligenceRouter = express.Router(); + +intelligenceRouter.use(requireAuth); +intelligenceRouter.use(requireIntelligenceApiEnabled); +intelligenceRouter.use(requireBrandAccess); + +intelligenceRouter.get('/:brand_key/intelligence/summary', handleSummary); +intelligenceRouter.get('/:brand_key/intelligence/store-timeseries', handleStoreTimeseries); +// ... etc +``` + +--- + +## 7. API Endpoints - Complete Specification + +### 7.1 GET /api/brands/:brand_key/intelligence/summary + +**Description:** Returns current summary metrics for a brand. + +**Parameters:** +| Param | Type | Required | Description | +|-------|------|----------|-------------| +| `brand_key` | string | Yes | cannabrands_id or cannabrands_slug | + +**SQL Query:** +```sql +-- Get latest metrics + comparisons +WITH latest AS ( + SELECT * FROM brand_daily_metrics + WHERE canonical_brand_id = $1 + ORDER BY metric_date DESC LIMIT 1 +), +comparisons AS ( + SELECT + metric_date, + total_stores, + total_skus, + skus_in_stock, + skus_on_promo + FROM brand_daily_metrics + WHERE canonical_brand_id = $1 + AND metric_date IN (CURRENT_DATE, CURRENT_DATE - 7, CURRENT_DATE - 30) +) +SELECT + l.*, + (SELECT total_stores FROM comparisons WHERE metric_date = CURRENT_DATE - 7) as stores_7d_ago, + (SELECT total_stores FROM comparisons WHERE metric_date = CURRENT_DATE - 30) as stores_30d_ago, + (SELECT total_skus FROM comparisons WHERE metric_date = CURRENT_DATE - 7) as skus_7d_ago, + (SELECT total_skus FROM comparisons WHERE metric_date = CURRENT_DATE - 30) as skus_30d_ago +FROM latest l; +``` + +**Response (Success - 200):** +```json +{ + "brand_key": "raw-garden", + "as_of": "2025-01-15T12:00:00.000Z", + "current": { + "total_stores": 127, + "total_skus": 543, + "skus_in_stock": 489, + "skus_on_promo": 34, + "avg_price": 42.50, + "price_range": { + "min": 25.00, + "max": 85.00 + } + }, + "changes": { + "week": { + "stores": 3, + "stores_percent": 2.42, + "skus": 12, + "skus_percent": 2.26 + }, + "month": { + "stores": 15, + "stores_percent": 13.39, + "skus": 67, + "skus_percent": 14.08 + } + } +} +``` + +**Response (No Data - 200):** +```json +{ + "brand_key": "new-brand", + "as_of": "2025-01-15T12:00:00.000Z", + "current": { + "total_stores": 0, + "total_skus": 0, + "skus_in_stock": 0, + "skus_on_promo": 0, + "avg_price": null, + "price_range": { + "min": null, + "max": null + } + }, + "changes": { + "week": { "stores": 0, "stores_percent": 0, "skus": 0, "skus_percent": 0 }, + "month": { "stores": 0, "stores_percent": 0, "skus": 0, "skus_percent": 0 } + } +} +``` + +**Error Responses:** +```json +// 403 - Access Denied +{ + "error": "Access denied", + "code": "BRAND_ACCESS_DENIED", + "message": "You do not have access to brand: raw-garden" +} + +// 404 - Brand Not Found +{ + "error": "Brand not found", + "code": "BRAND_NOT_FOUND", + "message": "No brand found with key: unknown-brand" +} + +// 404 - Feature Disabled +{ + "error": "Feature not available", + "code": "FEATURE_DISABLED" +} +``` + +--- + +### 7.2 GET /api/brands/:brand_key/intelligence/store-timeseries + +**Description:** Returns daily store/SKU counts for charting. + +**Parameters:** +| Param | Type | Required | Default | Description | +|-------|------|----------|---------|-------------| +| `brand_key` | string | Yes | - | cannabrands_id or cannabrands_slug | +| `days` | number | No | 30 | Number of days (max 365) | + +**SQL Query:** +```sql +SELECT + metric_date as date, + total_stores, + stores_added, + stores_removed, + total_skus, + skus_in_stock, + new_skus, + removed_skus +FROM brand_daily_metrics +WHERE canonical_brand_id = $1 + AND metric_date >= CURRENT_DATE - $2::integer +ORDER BY metric_date ASC; +``` + +**Response (Success - 200):** +```json +{ + "brand_key": "raw-garden", + "period": { + "days": 30, + "start": "2024-12-16", + "end": "2025-01-15" + }, + "timeseries": [ + { + "date": "2024-12-16", + "stores": { + "total": 112, + "added": 2, + "removed": 0, + "net": 2 + }, + "skus": { + "total": 476, + "in_stock": 423, + "new": 5, + "removed": 1, + "net": 4 + } + }, + { + "date": "2024-12-17", + "stores": { + "total": 114, + "added": 2, + "removed": 0, + "net": 2 + }, + "skus": { + "total": 482, + "in_stock": 430, + "new": 8, + "removed": 2, + "net": 6 + } + } + ] +} +``` + +**Response (No Data - 200):** +```json +{ + "brand_key": "new-brand", + "period": { + "days": 30, + "start": null, + "end": null + }, + "timeseries": [] +} +``` + +--- + +### 7.3 GET /api/brands/:brand_key/intelligence/store-changes + +**Description:** Returns recent store addition/removal events. + +**Parameters:** +| Param | Type | Required | Default | Description | +|-------|------|----------|---------|-------------| +| `brand_key` | string | Yes | - | cannabrands_id or cannabrands_slug | +| `limit` | number | No | 50 | Max results (max 200) | +| `event_type` | string | No | - | Filter: 'added', 'removed', 'reappeared' | + +**SQL Query:** +```sql +SELECT + bse.event_type, + bse.event_date, + bse.sku_count, + bse.notes, + s.id as store_id, + s.name as store_name, + s.slug as store_slug, + d.name as dispensary_name, + d.city, + d.state +FROM brand_store_events bse +JOIN stores s ON s.id = bse.store_id +JOIN dispensaries d ON d.id = bse.dispensary_id +WHERE bse.canonical_brand_id = $1 + AND ($2::text IS NULL OR bse.event_type = $2::brand_event_type) +ORDER BY bse.event_date DESC, bse.created_at DESC +LIMIT $3; +``` + +**Response (Success - 200):** +```json +{ + "brand_key": "raw-garden", + "total": 156, + "limit": 50, + "events": [ + { + "event_type": "added", + "date": "2025-01-14", + "store": { + "id": 42, + "name": "Green Thumb LA", + "slug": "green-thumb-la" + }, + "dispensary": { + "name": "Green Thumb Dispensary", + "location": "Los Angeles, CA" + }, + "sku_count": 12, + "notes": null + }, + { + "event_type": "removed", + "date": "2025-01-12", + "store": { + "id": 78, + "name": "MedMen Venice", + "slug": "medmen-venice" + }, + "dispensary": { + "name": "MedMen", + "location": "Venice, CA" + }, + "sku_count": null, + "notes": "Store closed permanently" + } + ] +} +``` + +--- + +### 7.4 GET /api/brands/:brand_key/intelligence/promo-timeseries + +**Description:** Returns daily promo/deal statistics. + +**Parameters:** +| Param | Type | Required | Default | Description | +|-------|------|----------|---------|-------------| +| `brand_key` | string | Yes | - | cannabrands_id or cannabrands_slug | +| `days` | number | No | 30 | Number of days (max 365) | + +**SQL Query:** +```sql +SELECT + metric_date as date, + total_promos, + percent_off_count, + dollar_off_count, + bogo_count, + bundle_count, + set_price_count, + other_count, + avg_discount_percent, + stores_with_promos +FROM brand_promo_daily_metrics +WHERE canonical_brand_id = $1 + AND metric_date >= CURRENT_DATE - $2::integer +ORDER BY metric_date ASC; +``` + +**Response (Success - 200):** +```json +{ + "brand_key": "raw-garden", + "period": { + "days": 30 + }, + "timeseries": [ + { + "date": "2024-12-16", + "total_promos": 45, + "breakdown": { + "percent_off": 28, + "dollar_off": 5, + "bogo": 8, + "bundle": 2, + "set_price": 1, + "other": 1 + }, + "avg_discount_percent": 18.5, + "stores_with_promos": 23 + }, + { + "date": "2024-12-17", + "total_promos": 52, + "breakdown": { + "percent_off": 32, + "dollar_off": 6, + "bogo": 10, + "bundle": 2, + "set_price": 1, + "other": 1 + }, + "avg_discount_percent": 19.2, + "stores_with_promos": 27 + } + ] +} +``` + +--- + +### 7.5 GET /api/brands/:brand_key/intelligence/stores + +**Description:** Returns all stores carrying this brand with current metrics. + +**Parameters:** +| Param | Type | Required | Default | Description | +|-------|------|----------|---------|-------------| +| `brand_key` | string | Yes | - | cannabrands_id or cannabrands_slug | +| `active` | boolean | No | true | Filter by active status | +| `state` | string | No | - | Filter by state (e.g., 'CA') | +| `limit` | number | No | 100 | Max results (max 500) | +| `offset` | number | No | 0 | Pagination offset | + +**SQL Query:** +```sql +SELECT + bsp.is_active, + bsp.first_seen_at, + bsp.last_seen_at, + bsp.removed_at, + bsp.active_sku_count, + bsp.in_stock_sku_count, + bsp.total_sku_count, + s.id as store_id, + s.name as store_name, + s.slug as store_slug, + d.name as dispensary_name, + d.city, + d.state, + d.dutchie_url +FROM brand_store_presence bsp +JOIN stores s ON s.id = bsp.store_id +JOIN dispensaries d ON d.id = bsp.dispensary_id +WHERE bsp.canonical_brand_id = $1 + AND ($2::boolean IS NULL OR bsp.is_active = $2) + AND ($3::text IS NULL OR d.state = $3) +ORDER BY bsp.active_sku_count DESC, s.name ASC +LIMIT $4 OFFSET $5; +``` + +**Response (Success - 200):** +```json +{ + "brand_key": "raw-garden", + "total": 127, + "limit": 100, + "offset": 0, + "filters": { + "active": true, + "state": null + }, + "stores": [ + { + "store": { + "id": 42, + "name": "Green Thumb LA", + "slug": "green-thumb-la" + }, + "dispensary": { + "name": "Green Thumb Dispensary", + "location": "Los Angeles, CA", + "dutchie_url": "https://dutchie.com/dispensary/green-thumb-la" + }, + "presence": { + "is_active": true, + "first_seen": "2024-06-15T00:00:00.000Z", + "last_seen": "2025-01-15T08:30:00.000Z", + "removed_at": null, + "tenure_days": 214 + }, + "skus": { + "active": 24, + "in_stock": 21, + "total": 28 + } + } + ] +} +``` + +--- + +### 7.6 GET /api/brands/:brand_key/intelligence/products + +**Description:** Returns all products for this brand with lifecycle data. + +**Parameters:** +| Param | Type | Required | Default | Description | +|-------|------|----------|---------|-------------| +| `brand_key` | string | Yes | - | cannabrands_id or cannabrands_slug | +| `store` | string | No | - | Filter by store slug | +| `status` | string | No | - | Filter: 'new', 'changed', 'stale', 'removed' | +| `in_stock` | boolean | No | - | Filter by stock status | +| `has_promo` | boolean | No | - | Filter by promo presence | +| `limit` | number | No | 100 | Max results (max 500) | +| `offset` | number | No | 0 | Pagination offset | + +**SQL Query:** +```sql +SELECT + sp.id, + sp.fingerprint, + sp.first_seen_at, + sp.last_seen_at, + sp.last_changed_at, + sp.removed_at, + sp.current_price, + sp.current_in_stock, + sp.current_special_text, + sp.current_special_type, + sp.current_special_value, + sp.current_special_data, + sp.change_status, + p.name as product_name, + p.brand as product_brand, + p.variant, + p.weight, + s.id as store_id, + s.name as store_name, + s.slug as store_slug, + d.city, + d.state +FROM store_products sp +JOIN products p ON p.id = sp.product_id +JOIN stores s ON s.id = sp.store_id +JOIN dispensaries d ON d.id = sp.dispensary_id +WHERE sp.canonical_brand_id = $1 + AND ($2::text IS NULL OR s.slug = $2) + AND ($3::product_change_type IS NULL OR sp.change_status = $3) + AND ($4::boolean IS NULL OR sp.current_in_stock = $4) + AND ($5::boolean IS NULL OR (sp.current_special_type != 'none') = $5) +ORDER BY sp.last_seen_at DESC +LIMIT $6 OFFSET $7; +``` + +**Response (Success - 200):** +```json +{ + "brand_key": "raw-garden", + "total": 543, + "limit": 100, + "offset": 0, + "filters": { + "store": null, + "status": null, + "in_stock": null, + "has_promo": null + }, + "products": [ + { + "id": 12345, + "fingerprint": "a1b2c3d4e5f6...", + "name": "Raw Garden Live Resin - Slurm OG", + "brand": "Raw Garden", + "variant": "1g", + "weight": "1g", + "store": { + "id": 42, + "name": "Green Thumb LA", + "slug": "green-thumb-la", + "location": "Los Angeles, CA" + }, + "lifecycle": { + "status": "unchanged", + "first_seen": "2024-08-10T00:00:00.000Z", + "last_seen": "2025-01-15T08:30:00.000Z", + "last_changed": "2025-01-10T12:00:00.000Z", + "removed_at": null + }, + "current": { + "price": 45.00, + "in_stock": true, + "special": { + "text": "20% off", + "type": "percent_off", + "value": 20, + "data": { + "percent": 20 + } + } + } + } + ] +} +``` + +--- + +## 8. Implementation Milestones + +### M0 - Schema Finalized + Migrations +- [ ] Create SQL migration file with all ENUMs +- [ ] Create SQL migration file with all tables +- [ ] Create SQL migration file with all indexes +- [ ] Create SQL migration file with v_brand_store_detail view +- [ ] Initialize feature_flags with all flags disabled +- [ ] Deploy migrations to staging +- [ ] Verify schema with `\d+` commands + +### M1 - Ingestion Engine Implemented + Idempotent +- [ ] Implement `parseSpecialText()` function +- [ ] Implement `generateFingerprint()` function +- [ ] Implement `resolveCanonicalBrand()` function +- [ ] Implement `ingestCrawlResults()` function with idempotency +- [ ] Implement `updateBrandStorePresence()` function +- [ ] Add unit tests for all functions +- [ ] Add integration test for full ingestion flow +- [ ] Hook into crawl completion handler (guarded by flag) +- [ ] Test with single store in staging + +### M2 - Daily Aggregation Engine Implemented +- [ ] Implement `aggregateBrandDailyMetrics()` function with idempotency +- [ ] Implement `aggregateBrandPromoMetrics()` function with idempotency +- [ ] Implement `runDailyBrandAggregation()` job runner +- [ ] Add cron schedule (2 AM daily) +- [ ] Add job monitoring/alerting +- [ ] Implement backfill script with idempotency +- [ ] Run backfill for last 30 days in staging + +### M3 - API Layer Implemented + Brand Scoping +- [ ] Implement `FeatureFlagService` +- [ ] Implement `resolveBrandKey()` function +- [ ] Implement `requireBrandAccess()` middleware +- [ ] Implement `requireIntelligenceApiEnabled()` middleware +- [ ] Implement all 6 API endpoints +- [ ] Add request validation with Zod/Joi +- [ ] Add response serialization +- [ ] Add API tests for all endpoints +- [ ] Add OpenAPI/Swagger documentation + +### M4 - Cannabrands App Integration +- [ ] Define JWT token structure with brand_key claim +- [ ] Set up test brand accounts in Cannabrands +- [ ] Map test brands to cannabrands_id +- [ ] Verify end-to-end API access from Cannabrands +- [ ] Document API for Cannabrands team +- [ ] Conduct integration review with Cannabrands team + +### M5 - Staging Load Test +- [ ] Generate synthetic data for 100+ brands +- [ ] Generate 30 days of metrics data +- [ ] Load test API endpoints (100 req/s) +- [ ] Profile slow queries and optimize +- [ ] Verify aggregation job completes in < 30 minutes +- [ ] Test rollback procedure (disable all flags) +- [ ] Test recovery procedure (re-enable flags) + +### M6 - Controlled Production Rollout +- [ ] Deploy migrations to production (flags OFF) +- [ ] Enable `INTELLIGENCE_INGEST_ENABLED` for 1 brand +- [ ] Monitor for 24 hours +- [ ] Enable `INTELLIGENCE_INGEST_ENABLED` for all brands +- [ ] Enable `BRAND_METRICS_ENABLED` +- [ ] Run initial backfill (7 days) +- [ ] Enable `INTELLIGENCE_API_ENABLED` +- [ ] Monitor API latency and error rates +- [ ] Announce availability to Cannabrands team + +--- + +## 9. Index Summary + +### Primary Tables + +| Table | Index | Columns | Notes | +|-------|-------|---------|-------| +| `feature_flags` | `idx_feature_flags_name` | `name` | Unique lookup | +| `crawl_runs` | `idx_crawl_runs_store_id` | `store_id` | | +| `crawl_runs` | `idx_crawl_runs_dispensary_id` | `dispensary_id` | | +| `crawl_runs` | `idx_crawl_runs_started_at` | `started_at DESC` | | +| `crawl_runs` | `idx_crawl_runs_store_started` | `store_id, started_at DESC` | | +| `crawl_runs` | `idx_crawl_runs_status` | `status` | Partial: running only | +| `canonical_brands` | `idx_canonical_brands_cannabrands_id` | `cannabrands_id` | Partial: NOT NULL | +| `canonical_brands` | `idx_canonical_brands_cannabrands_slug` | `cannabrands_slug` | Partial: NOT NULL | +| `canonical_brands` | `idx_canonical_brands_aliases` | `aliases` | GIN | +| `canonical_brands` | `idx_canonical_brands_name_lower` | `LOWER(name)` | | +| `brand_aliases` | `idx_brand_aliases_brand_id` | `canonical_brand_id` | | +| `brand_aliases` | `idx_brand_aliases_alias_lower` | `LOWER(alias)` | | +| `store_products` | `idx_store_products_store_id` | `store_id` | | +| `store_products` | `idx_store_products_dispensary_id` | `dispensary_id` | | +| `store_products` | `idx_store_products_product_id` | `product_id` | | +| `store_products` | `idx_store_products_brand_id` | `canonical_brand_id` | | +| `store_products` | `idx_store_products_fingerprint` | `fingerprint` | | +| `store_products` | `idx_store_products_status` | `change_status` | | +| `store_products` | `idx_store_products_store_status` | `store_id, change_status` | | +| `store_products` | `idx_store_products_active` | `store_id, canonical_brand_id` | Partial: active only | +| `store_products` | `idx_store_products_brand_store` | `canonical_brand_id, store_id` | | +| `store_products` | `idx_store_products_last_seen` | `last_seen_at DESC` | | +| `store_products` | `idx_store_products_first_seen` | `first_seen_at DESC` | | +| `store_product_snapshots` | `idx_snapshots_store_product_id` | `store_product_id` | | +| `store_product_snapshots` | `idx_snapshots_crawl_run_id` | `crawl_run_id` | | +| `store_product_snapshots` | `idx_snapshots_product_time` | `store_product_id, captured_at DESC` | | +| `store_product_snapshots` | `idx_snapshots_change_type` | `change_type` | | +| `store_product_snapshots` | `idx_snapshots_crawl_change` | `crawl_run_id, change_type` | | +| `brand_store_presence` | `idx_brand_presence_brand_id` | `canonical_brand_id` | | +| `brand_store_presence` | `idx_brand_presence_store_id` | `store_id` | | +| `brand_store_presence` | `idx_brand_presence_dispensary_id` | `dispensary_id` | | +| `brand_store_presence` | `idx_brand_presence_brand_store` | `canonical_brand_id, store_id` | | +| `brand_store_presence` | `idx_brand_presence_active` | `canonical_brand_id` | Partial: active only | +| `brand_store_presence` | `idx_brand_presence_store_active` | `store_id, is_active` | | +| `brand_daily_metrics` | `idx_brand_metrics_brand_id` | `canonical_brand_id` | | +| `brand_daily_metrics` | `idx_brand_metrics_date` | `metric_date DESC` | | +| `brand_daily_metrics` | `idx_brand_metrics_brand_date` | `canonical_brand_id, metric_date DESC` | | +| `brand_daily_metrics` | `idx_brand_metrics_date_brand` | `metric_date, canonical_brand_id` | | +| `brand_store_events` | `idx_brand_events_brand_id` | `canonical_brand_id` | | +| `brand_store_events` | `idx_brand_events_store_id` | `store_id` | | +| `brand_store_events` | `idx_brand_events_brand_date` | `canonical_brand_id, event_date DESC` | | +| `brand_store_events` | `idx_brand_events_store_date` | `store_id, event_date DESC` | | +| `brand_store_events` | `idx_brand_events_type` | `event_type` | | +| `brand_store_events` | `idx_brand_events_brand_type` | `canonical_brand_id, event_type, event_date DESC` | | +| `brand_store_events` | `idx_brand_events_crawl` | `crawl_run_id` | Partial: NOT NULL | +| `brand_promo_daily_metrics` | `idx_brand_promo_metrics_brand_id` | `canonical_brand_id` | | +| `brand_promo_daily_metrics` | `idx_brand_promo_metrics_date` | `metric_date DESC` | | +| `brand_promo_daily_metrics` | `idx_brand_promo_metrics_brand_date` | `canonical_brand_id, metric_date DESC` | | + +--- + +## 10. Rollback Procedures + +### Immediate API Disable +```sql +UPDATE feature_flags SET enabled = FALSE, updated_at = NOW() +WHERE name = 'INTELLIGENCE_API_ENABLED'; +``` +**Effect:** API returns 404, data continues ingesting. + +### Stop Ingestion +```sql +UPDATE feature_flags SET enabled = FALSE, updated_at = NOW() +WHERE name = 'INTELLIGENCE_INGEST_ENABLED'; +``` +**Effect:** Crawls stop populating intelligence tables. + +### Stop Aggregation +```sql +UPDATE feature_flags SET enabled = FALSE, updated_at = NOW() +WHERE name = 'BRAND_METRICS_ENABLED'; +``` +**Effect:** Daily aggregation job skips execution. + +### Full Disable +```sql +UPDATE feature_flags SET enabled = FALSE, updated_at = NOW() +WHERE name IN ( + 'INTELLIGENCE_ENABLED', + 'INTELLIGENCE_INGEST_ENABLED', + 'INTELLIGENCE_API_ENABLED', + 'BRAND_METRICS_ENABLED' +); +``` +**Effect:** All intelligence features disabled. + +### Recovery +```sql +-- Re-enable in order +UPDATE feature_flags SET enabled = TRUE, updated_at = NOW() +WHERE name = 'INTELLIGENCE_ENABLED'; + +UPDATE feature_flags SET enabled = TRUE, updated_at = NOW() +WHERE name = 'INTELLIGENCE_INGEST_ENABLED'; + +UPDATE feature_flags SET enabled = TRUE, updated_at = NOW() +WHERE name = 'BRAND_METRICS_ENABLED'; + +-- Enable API last +UPDATE feature_flags SET enabled = TRUE, updated_at = NOW() +WHERE name = 'INTELLIGENCE_API_ENABLED'; +``` + +### Data Cleanup (Nuclear - Use Only If Necessary) +```sql +-- Preserves schema, clears all intelligence data +TRUNCATE brand_promo_daily_metrics CASCADE; +TRUNCATE brand_daily_metrics CASCADE; +TRUNCATE brand_store_events CASCADE; +TRUNCATE brand_store_presence CASCADE; +TRUNCATE store_product_snapshots CASCADE; +TRUNCATE store_products CASCADE; +TRUNCATE crawl_runs CASCADE; +-- Keep canonical_brands as they may be linked to Cannabrands +``` + +--- + +## Document Version + +| Version | Date | Author | Changes | +|---------|------|--------|---------| +| 1.0 | 2025-01-15 | Claude | Initial complete specification | diff --git a/docs/STORE_API_SPECIFICATION.md b/docs/STORE_API_SPECIFICATION.md new file mode 100644 index 00000000..1c7d49b7 --- /dev/null +++ b/docs/STORE_API_SPECIFICATION.md @@ -0,0 +1,1958 @@ +# Store Menu API Specification + +## Overview + +This document specifies the **Store-Facing API** that powers the WordPress "Dutchie Plus replacement" plugin. This API is completely separate from the Brand Intelligence API. + +### Architecture Separation + +``` +┌─────────────────────────────────────────────────────────────────┐ +│ CRAWLER + CANONICAL TABLES │ +│ (Single Source of Truth) │ +│ │ +│ ┌──────────────┐ ┌──────────────┐ ┌──────────────────────┐ │ +│ │ products │ │ stores │ │ store_products │ │ +│ │ table │ │ table │ │ (current state) │ │ +│ └──────────────┘ └──────────────┘ └──────────────────────┘ │ +└─────────────────────────────┬───────────────────────────────────┘ + │ + ┌─────────────────────┴─────────────────────┐ + ▼ ▼ +┌───────────────────────────────┐ ┌───────────────────────────────┐ +│ BRAND INTELLIGENCE API │ │ STORE MENU API │ +│ (Cannabrands App) │ │ (WordPress Plugin) │ +├───────────────────────────────┤ ├───────────────────────────────┤ +│ Path: /api/brands/:key/... │ │ Path: /api/stores/:key/... │ +│ Auth: JWT + brand_key claim │ │ Auth: X-Store-API-Key header │ +│ Scope: Cross-store analytics │ │ Scope: Single store only │ +│ │ │ │ +│ READS FROM: │ │ READS FROM: │ +│ • brand_daily_metrics │ │ • products │ +│ • brand_promo_daily_metrics │ │ • categories │ +│ • brand_store_events │ │ • stores │ +│ • brand_store_presence │ │ • (NO analytics tables) │ +│ • store_product_snapshots │ │ │ +└───────────────────────────────┘ └───────────────────────────────┘ +``` + +### Key Principles + +1. **Read-only** - No writes to the database +2. **Store-scoped** - Each request is scoped to a single store +3. **Current state only** - No historical analytics +4. **Cache-friendly** - Deterministic responses for given inputs +5. **WordPress-optimized** - Designed for transient caching + +--- + +## 1. Authentication + +### 1.1 Store API Keys + +Each store gets one or more API keys stored in a dedicated table: + +```sql +CREATE TABLE store_api_keys ( + id SERIAL PRIMARY KEY, + store_id INTEGER NOT NULL REFERENCES stores(id) ON DELETE CASCADE, + api_key VARCHAR(64) NOT NULL, + name VARCHAR(100) NOT NULL DEFAULT 'WordPress Plugin', + is_active BOOLEAN NOT NULL DEFAULT TRUE, + rate_limit_hour INTEGER NOT NULL DEFAULT 1000, + expires_at TIMESTAMPTZ, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + last_used_at TIMESTAMPTZ, + + CONSTRAINT uq_store_api_keys_key UNIQUE (api_key) +); + +CREATE INDEX idx_store_api_keys_key ON store_api_keys(api_key) WHERE is_active = TRUE; +CREATE INDEX idx_store_api_keys_store ON store_api_keys(store_id); +``` + +### 1.2 Request Authentication + +**Header:** `X-Store-API-Key: sk_live_abc123def456...` + +**Resolution Flow:** + +``` +Request: GET /api/stores/deeply-rooted/menu +Header: X-Store-API-Key: sk_live_abc123def456 + + ↓ +┌───────────────────────────────────────────────────┐ +│ 1. Lookup API key in store_api_keys │ +│ WHERE api_key = 'sk_live_abc123def456' │ +│ AND is_active = TRUE │ +│ AND (expires_at IS NULL OR expires_at > NOW())│ +└───────────────────────────────────────────────────┘ + ↓ +┌───────────────────────────────────────────────────┐ +│ 2. Verify store_key matches the API key's store │ +│ API key's store_id → stores.slug must match │ +│ the :store_key in the URL path │ +└───────────────────────────────────────────────────┘ + ↓ +┌───────────────────────────────────────────────────┐ +│ 3. Rate limit check (1000 req/hour default) │ +└───────────────────────────────────────────────────┘ + ↓ + Request proceeds +``` + +### 1.3 Store Key Resolution + +The `store_key` path parameter can be: +- Store slug (e.g., `deeply-rooted-sacramento`) +- Store ID (e.g., `1`) + +```typescript +async function resolveStoreKey(db: Pool, storeKey: string): Promise { + // Try as numeric ID first + if (/^\d+$/.test(storeKey)) { + const byId = await db.query( + `SELECT id FROM stores WHERE id = $1`, + [parseInt(storeKey)] + ); + if (byId.rows.length > 0) return byId.rows[0].id; + } + + // Try as slug + const bySlug = await db.query( + `SELECT id FROM stores WHERE slug = $1`, + [storeKey] + ); + if (bySlug.rows.length > 0) return bySlug.rows[0].id; + + return null; +} +``` + +### 1.4 Security Guarantees + +- API key only grants access to its associated store +- No cross-store data leakage possible +- Rate limiting prevents abuse +- Keys can be rotated/revoked without code changes + +--- + +## 2. Canonical Data Types + +### 2.1 ProductForStore Object + +This is the canonical product representation used across all store endpoints: + +```typescript +interface ProductForStore { + // Identifiers + product_id: number; // products.id (canonical product) + store_product_id: number | null; // store_products.id (if using intelligence layer) + slug: string; // URL-safe identifier + + // Basic info + name: string; // Product name (without brand prefix) + full_name: string; // Full name including brand + brand_name: string | null; // Brand name + brand_id: number | null; // canonical_brands.id (if mapped) + + // Classification + category: CategoryInfo; + subcategory: CategoryInfo | null; + strain_type: 'indica' | 'sativa' | 'hybrid' | 'cbd' | null; + + // Sizing + weight: string | null; // Display weight (e.g., "1g", "3.5g", "1oz") + weight_grams: number | null; // Numeric grams for sorting + + // Cannabinoids + thc_percent: number | null; + thc_range: string | null; // "25-30%" if range + cbd_percent: number | null; + + // Availability + in_stock: boolean; + + // Pricing (all in cents for precision) + price_cents: number; // Current selling price + regular_price_cents: number | null; // Pre-discount price (if on special) + + // Specials + is_on_special: boolean; + special: SpecialInfo | null; + + // Media + image_url: string | null; // Primary image URL + images: ImageSet; + + // Description + description: string | null; + + // Metadata + terpenes: string[] | null; + effects: string[] | null; + flavors: string[] | null; + + // Links + dutchie_url: string | null; + + // Timestamps + last_updated_at: string; // ISO 8601 +} + +interface CategoryInfo { + id: number; + name: string; + slug: string; +} + +interface SpecialInfo { + type: 'percent_off' | 'dollar_off' | 'bogo' | 'bundle' | 'set_price' | 'other'; + text: string; // Raw special text for display + badge_text: string; // Short badge (e.g., "20% OFF", "BOGO") + value: number | null; // Discount value (percent or dollars) + savings_cents: number | null; // Calculated savings in cents +} + +interface ImageSet { + thumbnail: string | null; + medium: string | null; + full: string | null; +} +``` + +### 2.2 Example ProductForStore JSON + +```json +{ + "product_id": 12345, + "store_product_id": 67890, + "slug": "thunder-bud-1g-pre-roll", + "name": "1g Pre-Roll", + "full_name": "Thunder Bud 1g Pre-Roll", + "brand_name": "Thunder Bud", + "brand_id": 22, + "category": { + "id": 3, + "name": "Pre-Rolls", + "slug": "pre-rolls" + }, + "subcategory": { + "id": 15, + "name": "Singles", + "slug": "singles" + }, + "strain_type": "hybrid", + "weight": "1g", + "weight_grams": 1.0, + "thc_percent": 27.5, + "thc_range": null, + "cbd_percent": 0.1, + "in_stock": true, + "price_cents": 1200, + "regular_price_cents": 1500, + "is_on_special": true, + "special": { + "type": "percent_off", + "text": "20% off all Thunder Bud pre-rolls", + "badge_text": "20% OFF", + "value": 20, + "savings_cents": 300 + }, + "image_url": "https://images.dutchie.com/abc123/medium.jpg", + "images": { + "thumbnail": "https://images.dutchie.com/abc123/thumb.jpg", + "medium": "https://images.dutchie.com/abc123/medium.jpg", + "full": "https://images.dutchie.com/abc123/full.jpg" + }, + "description": "Premium pre-roll featuring Alien Marker strain. Hand-rolled with care.", + "terpenes": ["Limonene", "Myrcene"], + "effects": ["Relaxed", "Happy"], + "flavors": ["Citrus", "Earthy"], + "dutchie_url": "https://dutchie.com/dispensary/deeply-rooted/product/thunder-bud-1g-pre-roll", + "last_updated_at": "2025-02-28T15:03:00Z" +} +``` + +### 2.3 CategoryForStore Object + +```typescript +interface CategoryForStore { + id: number; + name: string; + slug: string; + icon: string | null; // Icon identifier + product_count: number; // Total products in category + in_stock_count: number; // Products currently in stock + on_special_count: number; // Products on special + children: CategoryForStore[]; // Subcategories +} +``` + +--- + +## 3. API Endpoints + +### 3.1 GET /api/stores/:store_key/menu + +Returns the full product menu for a store. + +**Path Parameters:** +| Param | Type | Description | +|-------|------|-------------| +| `store_key` | string | Store slug or ID | + +**Query Parameters:** +| Param | Type | Required | Default | Description | +|-------|------|----------|---------|-------------| +| `category` | string | No | - | Filter by category slug (e.g., `flower`, `pre-rolls`) | +| `brand` | string | No | - | Filter by brand name (partial match) | +| `search` | string | No | - | Free text search on product name | +| `in_stock` | boolean | No | `true` | Filter by stock status | +| `sort` | string | No | `name_asc` | Sort order (see below) | +| `page` | number | No | `1` | Page number (1-indexed) | +| `per_page` | number | No | `50` | Products per page (max 200) | + +**Sort Options:** +- `name_asc` - Name A-Z (default) +- `name_desc` - Name Z-A +- `price_asc` - Price low to high +- `price_desc` - Price high to low +- `thc_desc` - THC% high to low +- `newest` - Most recently updated first + +**Response:** + +```json +{ + "store": { + "id": 1, + "name": "Deeply Rooted", + "slug": "deeply-rooted-sacramento", + "city": "Sacramento", + "state": "CA", + "logo_url": null, + "dutchie_url": "https://dutchie.com/dispensary/deeply-rooted" + }, + "meta": { + "total_products": 342, + "total_pages": 7, + "current_page": 1, + "per_page": 50, + "filters_applied": { + "category": null, + "brand": null, + "search": null, + "in_stock": true + }, + "sort": "name_asc", + "generated_at": "2025-02-28T15:00:00Z", + "cache_ttl_seconds": 300 + }, + "products": [ + { + "product_id": 12345, + "store_product_id": 67890, + "slug": "thunder-bud-1g-pre-roll", + "name": "1g Pre-Roll", + "full_name": "Thunder Bud 1g Pre-Roll", + "brand_name": "Thunder Bud", + "brand_id": 22, + "category": { + "id": 3, + "name": "Pre-Rolls", + "slug": "pre-rolls" + }, + "subcategory": null, + "strain_type": "hybrid", + "weight": "1g", + "weight_grams": 1.0, + "thc_percent": 27.5, + "thc_range": null, + "cbd_percent": 0.1, + "in_stock": true, + "price_cents": 1200, + "regular_price_cents": 1500, + "is_on_special": true, + "special": { + "type": "percent_off", + "text": "20% off all Thunder Bud pre-rolls", + "badge_text": "20% OFF", + "value": 20, + "savings_cents": 300 + }, + "image_url": "https://images.dutchie.com/abc123/medium.jpg", + "images": { + "thumbnail": "https://images.dutchie.com/abc123/thumb.jpg", + "medium": "https://images.dutchie.com/abc123/medium.jpg", + "full": "https://images.dutchie.com/abc123/full.jpg" + }, + "description": "Premium pre-roll featuring Alien Marker strain.", + "terpenes": ["Limonene", "Myrcene"], + "effects": ["Relaxed", "Happy"], + "flavors": ["Citrus", "Earthy"], + "dutchie_url": "https://dutchie.com/dispensary/deeply-rooted/product/thunder-bud-1g-pre-roll", + "last_updated_at": "2025-02-28T15:03:00Z" + } + // ... more products + ] +} +``` + +**SQL Query:** + +```sql +-- Main menu query +SELECT + p.id AS product_id, + p.slug, + p.name, + COALESCE(p.brand || ' ' || p.name, p.name) AS full_name, + p.brand AS brand_name, + cb.id AS brand_id, + p.category_id, + c.name AS category_name, + c.slug AS category_slug, + pc.id AS parent_category_id, + pc.name AS parent_category_name, + pc.slug AS parent_category_slug, + p.strain_type, + p.weight, + -- Parse weight to grams for sorting + CASE + WHEN p.weight ~ '^\d+(\.\d+)?g$' THEN CAST(REGEXP_REPLACE(p.weight, 'g$', '') AS NUMERIC) + WHEN p.weight ~ '^\d+(\.\d+)?oz$' THEN CAST(REGEXP_REPLACE(p.weight, 'oz$', '') AS NUMERIC) * 28.35 + ELSE NULL + END AS weight_grams, + p.thc_percentage AS thc_percent, + p.cbd_percentage AS cbd_percent, + p.in_stock, + -- Current price in cents + ROUND(COALESCE(p.price, 0) * 100)::INTEGER AS price_cents, + -- Regular price (if on special) + CASE + WHEN p.original_price IS NOT NULL AND p.original_price > p.price + THEN ROUND(p.original_price * 100)::INTEGER + ELSE NULL + END AS regular_price_cents, + -- Special detection + (p.special_text IS NOT NULL AND p.special_text != '') AS is_on_special, + p.special_text, + -- Images + p.image_url_full, + p.image_url, + p.thumbnail_path, + p.medium_path, + -- Description & metadata + p.description, + p.metadata, + p.dutchie_url, + p.last_seen_at AS last_updated_at +FROM products p +LEFT JOIN categories c ON c.id = p.category_id +LEFT JOIN categories pc ON pc.id = c.parent_id +LEFT JOIN canonical_brands cb ON LOWER(cb.name) = LOWER(p.brand) +WHERE p.store_id = $1 + AND ($2::TEXT IS NULL OR c.slug = $2 OR pc.slug = $2) -- category filter + AND ($3::TEXT IS NULL OR p.brand ILIKE '%' || $3 || '%') -- brand filter + AND ($4::TEXT IS NULL OR p.name ILIKE '%' || $4 || '%') -- search filter + AND ($5::BOOLEAN IS NULL OR p.in_stock = $5) -- in_stock filter +ORDER BY + CASE WHEN $6 = 'name_asc' THEN p.name END ASC, + CASE WHEN $6 = 'name_desc' THEN p.name END DESC, + CASE WHEN $6 = 'price_asc' THEN p.price END ASC, + CASE WHEN $6 = 'price_desc' THEN p.price END DESC, + CASE WHEN $6 = 'thc_desc' THEN p.thc_percentage END DESC NULLS LAST, + CASE WHEN $6 = 'newest' THEN p.last_seen_at END DESC, + p.name ASC -- Secondary sort for stability +LIMIT $7 OFFSET $8; + +-- Count query for pagination +SELECT COUNT(*) AS total +FROM products p +LEFT JOIN categories c ON c.id = p.category_id +LEFT JOIN categories pc ON pc.id = c.parent_id +WHERE p.store_id = $1 + AND ($2::TEXT IS NULL OR c.slug = $2 OR pc.slug = $2) + AND ($3::TEXT IS NULL OR p.brand ILIKE '%' || $3 || '%') + AND ($4::TEXT IS NULL OR p.name ILIKE '%' || $4 || '%') + AND ($5::BOOLEAN IS NULL OR p.in_stock = $5); +``` + +--- + +### 3.2 GET /api/stores/:store_key/specials + +Returns only products currently on special. + +**Query Parameters:** +| Param | Type | Required | Default | Description | +|-------|------|----------|---------|-------------| +| `category` | string | No | - | Filter by category slug | +| `brand` | string | No | - | Filter by brand name | +| `special_type` | string | No | - | Filter: `percent_off`, `dollar_off`, `bogo`, `bundle` | +| `sort` | string | No | `savings_desc` | Sort order | +| `limit` | number | No | `50` | Max products (max 100) | + +**Sort Options:** +- `savings_desc` - Highest savings first (default) +- `savings_percent_desc` - Highest % discount first +- `price_asc` - Price low to high +- `name_asc` - Name A-Z + +**Response:** + +```json +{ + "store": { + "id": 1, + "name": "Deeply Rooted", + "slug": "deeply-rooted-sacramento" + }, + "meta": { + "total_specials": 34, + "filters_applied": { + "category": null, + "brand": null, + "special_type": null + }, + "limit": 50, + "generated_at": "2025-02-28T15:00:00Z", + "cache_ttl_seconds": 300 + }, + "specials": [ + { + "product_id": 12345, + "store_product_id": 67890, + "slug": "thunder-bud-1g-pre-roll", + "name": "1g Pre-Roll", + "full_name": "Thunder Bud 1g Pre-Roll", + "brand_name": "Thunder Bud", + "brand_id": 22, + "category": { + "id": 3, + "name": "Pre-Rolls", + "slug": "pre-rolls" + }, + "subcategory": null, + "strain_type": "hybrid", + "weight": "1g", + "weight_grams": 1.0, + "thc_percent": 27.5, + "thc_range": null, + "cbd_percent": 0.1, + "in_stock": true, + "price_cents": 1200, + "regular_price_cents": 1500, + "is_on_special": true, + "special": { + "type": "percent_off", + "text": "20% off all Thunder Bud pre-rolls", + "badge_text": "20% OFF", + "value": 20, + "savings_cents": 300 + }, + "image_url": "https://images.dutchie.com/abc123/medium.jpg", + "images": { + "thumbnail": "https://images.dutchie.com/abc123/thumb.jpg", + "medium": "https://images.dutchie.com/abc123/medium.jpg", + "full": "https://images.dutchie.com/abc123/full.jpg" + }, + "description": "Premium pre-roll featuring Alien Marker strain.", + "terpenes": null, + "effects": null, + "flavors": null, + "dutchie_url": "https://dutchie.com/dispensary/deeply-rooted/product/thunder-bud-1g-pre-roll", + "last_updated_at": "2025-02-28T15:03:00Z" + } + // ... more specials + ], + "summary": { + "by_type": { + "percent_off": 18, + "dollar_off": 5, + "bogo": 8, + "bundle": 2, + "other": 1 + }, + "by_category": [ + { "slug": "flower", "name": "Flower", "count": 12 }, + { "slug": "concentrates", "name": "Concentrates", "count": 10 }, + { "slug": "pre-rolls", "name": "Pre-Rolls", "count": 8 } + ], + "avg_savings_percent": 22.5, + "total_savings_available_cents": 15600 + } +} +``` + +**SQL Query:** + +```sql +SELECT + p.id AS product_id, + p.slug, + p.name, + COALESCE(p.brand || ' ' || p.name, p.name) AS full_name, + p.brand AS brand_name, + cb.id AS brand_id, + c.id AS category_id, + c.name AS category_name, + c.slug AS category_slug, + p.strain_type, + p.weight, + p.thc_percentage AS thc_percent, + p.cbd_percentage AS cbd_percent, + p.in_stock, + ROUND(COALESCE(p.price, 0) * 100)::INTEGER AS price_cents, + ROUND(COALESCE(p.original_price, p.price) * 100)::INTEGER AS regular_price_cents, + TRUE AS is_on_special, + p.special_text, + p.image_url_full, + p.image_url, + p.thumbnail_path, + p.medium_path, + p.description, + p.metadata, + p.dutchie_url, + p.last_seen_at AS last_updated_at, + -- Calculated savings + ROUND((COALESCE(p.original_price, p.price) - p.price) * 100)::INTEGER AS savings_cents, + CASE + WHEN p.original_price > 0 + THEN ROUND(((p.original_price - p.price) / p.original_price) * 100, 1) + ELSE 0 + END AS savings_percent +FROM products p +LEFT JOIN categories c ON c.id = p.category_id +LEFT JOIN canonical_brands cb ON LOWER(cb.name) = LOWER(p.brand) +WHERE p.store_id = $1 + AND p.special_text IS NOT NULL + AND p.special_text != '' + AND p.in_stock = TRUE -- Only show in-stock specials + AND ($2::TEXT IS NULL OR c.slug = $2) + AND ($3::TEXT IS NULL OR p.brand ILIKE '%' || $3 || '%') +ORDER BY + CASE WHEN $4 = 'savings_desc' THEN (COALESCE(p.original_price, p.price) - p.price) END DESC, + CASE WHEN $4 = 'savings_percent_desc' THEN + CASE WHEN p.original_price > 0 THEN ((p.original_price - p.price) / p.original_price) ELSE 0 END + END DESC, + CASE WHEN $4 = 'price_asc' THEN p.price END ASC, + CASE WHEN $4 = 'name_asc' THEN p.name END ASC, + p.name ASC +LIMIT $5; +``` + +--- + +### 3.3 GET /api/stores/:store_key/products + +General paginated product listing with full filter support. Used for search results and custom layouts. + +**Query Parameters:** +| Param | Type | Required | Default | Description | +|-------|------|----------|---------|-------------| +| `category` | string | No | - | Filter by category slug | +| `brand` | string | No | - | Filter by brand name | +| `search` | string | No | - | Free text search | +| `in_stock` | boolean | No | - | Filter by stock status (null = all) | +| `specials_only` | boolean | No | `false` | Only show products on special | +| `strain_type` | string | No | - | Filter: `indica`, `sativa`, `hybrid`, `cbd` | +| `min_thc` | number | No | - | Minimum THC% | +| `max_price` | number | No | - | Maximum price in dollars | +| `sort` | string | No | `name_asc` | Sort order | +| `page` | number | No | `1` | Page number | +| `per_page` | number | No | `24` | Products per page (max 100) | + +**Response:** + +```json +{ + "store": { + "id": 1, + "name": "Deeply Rooted", + "slug": "deeply-rooted-sacramento" + }, + "meta": { + "total_products": 45, + "total_pages": 2, + "current_page": 1, + "per_page": 24, + "filters_applied": { + "category": "flower", + "brand": null, + "search": null, + "in_stock": true, + "specials_only": false, + "strain_type": null, + "min_thc": null, + "max_price": null + }, + "sort": "thc_desc", + "generated_at": "2025-02-28T15:00:00Z", + "cache_ttl_seconds": 300 + }, + "products": [ + // ... array of ProductForStore objects + ], + "facets": { + "categories": [ + { "slug": "indoor", "name": "Indoor", "count": 25 }, + { "slug": "outdoor", "name": "Outdoor", "count": 15 }, + { "slug": "greenhouse", "name": "Greenhouse", "count": 5 } + ], + "brands": [ + { "name": "Raw Garden", "count": 12 }, + { "name": "Connected", "count": 8 }, + { "name": "Alien Labs", "count": 5 } + ], + "strain_types": [ + { "type": "hybrid", "count": 20 }, + { "type": "indica", "count": 15 }, + { "type": "sativa", "count": 10 } + ], + "price_range": { + "min_cents": 1500, + "max_cents": 8500 + }, + "thc_range": { + "min": 15.0, + "max": 35.0 + } + } +} +``` + +--- + +### 3.4 GET /api/stores/:store_key/categories + +Returns the category tree with product counts. + +**Response:** + +```json +{ + "store": { + "id": 1, + "name": "Deeply Rooted", + "slug": "deeply-rooted-sacramento" + }, + "meta": { + "total_categories": 12, + "generated_at": "2025-02-28T15:00:00Z", + "cache_ttl_seconds": 600 + }, + "categories": [ + { + "id": 1, + "name": "Flower", + "slug": "flower", + "icon": "flower", + "product_count": 85, + "in_stock_count": 72, + "on_special_count": 8, + "children": [ + { + "id": 10, + "name": "Indoor", + "slug": "indoor", + "icon": null, + "product_count": 45, + "in_stock_count": 40, + "on_special_count": 4, + "children": [] + }, + { + "id": 11, + "name": "Outdoor", + "slug": "outdoor", + "icon": null, + "product_count": 25, + "in_stock_count": 20, + "on_special_count": 2, + "children": [] + }, + { + "id": 12, + "name": "Greenhouse", + "slug": "greenhouse", + "icon": null, + "product_count": 15, + "in_stock_count": 12, + "on_special_count": 2, + "children": [] + } + ] + }, + { + "id": 2, + "name": "Pre-Rolls", + "slug": "pre-rolls", + "icon": "joint", + "product_count": 42, + "in_stock_count": 38, + "on_special_count": 5, + "children": [ + { + "id": 20, + "name": "Singles", + "slug": "singles", + "icon": null, + "product_count": 28, + "in_stock_count": 25, + "on_special_count": 3, + "children": [] + }, + { + "id": 21, + "name": "Packs", + "slug": "packs", + "icon": null, + "product_count": 14, + "in_stock_count": 13, + "on_special_count": 2, + "children": [] + } + ] + }, + { + "id": 3, + "name": "Vapes", + "slug": "vapes", + "icon": "vape", + "product_count": 65, + "in_stock_count": 58, + "on_special_count": 7, + "children": [] + }, + { + "id": 4, + "name": "Concentrates", + "slug": "concentrates", + "icon": "dab", + "product_count": 48, + "in_stock_count": 42, + "on_special_count": 6, + "children": [] + }, + { + "id": 5, + "name": "Edibles", + "slug": "edibles", + "icon": "cookie", + "product_count": 56, + "in_stock_count": 50, + "on_special_count": 4, + "children": [] + }, + { + "id": 6, + "name": "Tinctures", + "slug": "tinctures", + "icon": "dropper", + "product_count": 18, + "in_stock_count": 16, + "on_special_count": 2, + "children": [] + }, + { + "id": 7, + "name": "Topicals", + "slug": "topicals", + "icon": "lotion", + "product_count": 12, + "in_stock_count": 10, + "on_special_count": 1, + "children": [] + }, + { + "id": 8, + "name": "Accessories", + "slug": "accessories", + "icon": "gear", + "product_count": 16, + "in_stock_count": 14, + "on_special_count": 1, + "children": [] + } + ] +} +``` + +**SQL Query:** + +```sql +-- Get categories with counts for a store +WITH category_counts AS ( + SELECT + c.id, + c.name, + c.slug, + c.parent_id, + c.icon, + COUNT(p.id) AS product_count, + COUNT(p.id) FILTER (WHERE p.in_stock = TRUE) AS in_stock_count, + COUNT(p.id) FILTER (WHERE p.special_text IS NOT NULL AND p.special_text != '') AS on_special_count + FROM categories c + LEFT JOIN products p ON p.category_id = c.id AND p.store_id = $1 + WHERE c.store_id = $1 OR c.store_id IS NULL -- Global + store-specific categories + GROUP BY c.id, c.name, c.slug, c.parent_id, c.icon + HAVING COUNT(p.id) > 0 -- Only categories with products +) +SELECT * FROM category_counts +ORDER BY + CASE WHEN parent_id IS NULL THEN 0 ELSE 1 END, -- Parents first + product_count DESC, + name ASC; +``` + +--- + +### 3.5 GET /api/stores/:store_key/brands + +Returns all brands available at the store. + +**Response:** + +```json +{ + "store": { + "id": 1, + "name": "Deeply Rooted", + "slug": "deeply-rooted-sacramento" + }, + "meta": { + "total_brands": 48, + "generated_at": "2025-02-28T15:00:00Z", + "cache_ttl_seconds": 600 + }, + "brands": [ + { + "name": "Raw Garden", + "slug": "raw-garden", + "brand_id": 15, + "product_count": 24, + "in_stock_count": 21, + "on_special_count": 3, + "categories": ["Concentrates", "Vapes"], + "price_range": { + "min_cents": 2500, + "max_cents": 6500 + } + }, + { + "name": "Stiiizy", + "slug": "stiiizy", + "brand_id": 8, + "product_count": 18, + "in_stock_count": 15, + "on_special_count": 0, + "categories": ["Vapes", "Flower", "Edibles"], + "price_range": { + "min_cents": 2000, + "max_cents": 5500 + } + }, + { + "name": "Thunder Bud", + "slug": "thunder-bud", + "brand_id": 22, + "product_count": 12, + "in_stock_count": 10, + "on_special_count": 4, + "categories": ["Flower", "Pre-Rolls"], + "price_range": { + "min_cents": 1200, + "max_cents": 4500 + } + } + // ... more brands + ] +} +``` + +--- + +### 3.6 GET /api/stores/:store_key/product/:product_slug + +Returns detailed information for a single product. + +**Response:** + +```json +{ + "store": { + "id": 1, + "name": "Deeply Rooted", + "slug": "deeply-rooted-sacramento" + }, + "product": { + "product_id": 12345, + "store_product_id": 67890, + "slug": "thunder-bud-1g-pre-roll", + "name": "1g Pre-Roll", + "full_name": "Thunder Bud 1g Pre-Roll", + "brand_name": "Thunder Bud", + "brand_id": 22, + "category": { + "id": 3, + "name": "Pre-Rolls", + "slug": "pre-rolls" + }, + "subcategory": { + "id": 15, + "name": "Singles", + "slug": "singles" + }, + "strain_type": "hybrid", + "strain_name": "Alien Marker", + "weight": "1g", + "weight_grams": 1.0, + "thc_percent": 27.5, + "thc_range": null, + "cbd_percent": 0.1, + "in_stock": true, + "price_cents": 1200, + "regular_price_cents": 1500, + "is_on_special": true, + "special": { + "type": "percent_off", + "text": "20% off all Thunder Bud pre-rolls", + "badge_text": "20% OFF", + "value": 20, + "savings_cents": 300 + }, + "image_url": "https://images.dutchie.com/abc123/medium.jpg", + "images": { + "thumbnail": "https://images.dutchie.com/abc123/thumb.jpg", + "medium": "https://images.dutchie.com/abc123/medium.jpg", + "full": "https://images.dutchie.com/abc123/full.jpg" + }, + "description": "Premium pre-roll featuring Alien Marker strain. Hand-rolled with care using top-shelf flower. Perfect for a smooth, balanced experience.", + "terpenes": [ + { "name": "Limonene", "percentage": 1.2 }, + { "name": "Myrcene", "percentage": 0.8 }, + { "name": "Caryophyllene", "percentage": 0.5 } + ], + "effects": ["Relaxed", "Happy", "Euphoric", "Uplifted"], + "flavors": ["Citrus", "Earthy", "Pine"], + "lineage": "Unknown lineage", + "dutchie_url": "https://dutchie.com/dispensary/deeply-rooted/product/thunder-bud-1g-pre-roll", + "last_updated_at": "2025-02-28T15:03:00Z" + }, + "related_products": [ + { + "product_id": 12346, + "slug": "thunder-bud-3-5g-flower", + "name": "3.5g Flower", + "full_name": "Thunder Bud 3.5g Flower", + "brand_name": "Thunder Bud", + "category": { "id": 1, "name": "Flower", "slug": "flower" }, + "price_cents": 3500, + "image_url": "https://images.dutchie.com/def456/medium.jpg", + "in_stock": true, + "is_on_special": true + }, + { + "product_id": 12400, + "slug": "alien-labs-pre-roll-1g", + "name": "1g Pre-Roll", + "full_name": "Alien Labs 1g Pre-Roll", + "brand_name": "Alien Labs", + "category": { "id": 3, "name": "Pre-Rolls", "slug": "pre-rolls" }, + "price_cents": 1800, + "image_url": "https://images.dutchie.com/ghi789/medium.jpg", + "in_stock": true, + "is_on_special": false + } + ] +} +``` + +--- + +## 4. Specials Detection (API Layer) + +> **IMPORTANT**: The crawler is FROZEN. All specials detection happens in the API layer, not in the crawler. +> See [CRAWL_OPERATIONS.md](./CRAWL_OPERATIONS.md) for the frozen crawler policy. + +### 4.1 Why API-Layer Detection? + +The crawler captures special information but doesn't always set the `is_special` flag correctly: + +| Data Source | Current State | Products | +|-------------|---------------|----------| +| `is_special = true` | Always false | 0 | +| "Special Offer" in product name | Embedded in name text | 325 | +| `sale_price < regular_price` | Price comparison | 4 | +| `special_text` field | Empty | 0 | + +Since the crawler is frozen, we detect specials at query time using multiple signals. + +### 4.2 Specials Detection Rules (Priority Order) + +The API determines `is_on_special = true` if ANY of these conditions are met: + +```sql +-- Computed is_on_special in API queries +CASE + -- Rule 1: "Special Offer" appears in the product name + WHEN p.name ILIKE '%Special Offer%' THEN TRUE + + -- Rule 2: sale_price is less than regular_price + WHEN p.sale_price IS NOT NULL + AND p.regular_price IS NOT NULL + AND p.sale_price::numeric < p.regular_price::numeric THEN TRUE + + -- Rule 3: price is less than original_price + WHEN p.price IS NOT NULL + AND p.original_price IS NOT NULL + AND p.price::numeric < p.original_price::numeric THEN TRUE + + -- Rule 4: special_text field is populated (if crawler sets it) + WHEN p.special_text IS NOT NULL AND p.special_text != '' THEN TRUE + + ELSE FALSE +END AS is_on_special +``` + +### 4.3 Clean Product Name Function + +Strip "Special Offer" from the display name to avoid redundancy: + +```typescript +function cleanProductName(rawName: string): string { + return rawName + .replace(/\s*Special Offer\s*$/i, '') // Remove trailing "Special Offer" + .replace(/\s+$/, '') // Trim trailing whitespace + .trim(); +} + +// Example: +// Input: "Canamo Cured Batter | CuracaoCanamo ConcentratesHybridTHC: 77.66%CBD: 0.15%Special Offer" +// Output: "Canamo Cured Batter | CuracaoCanamo ConcentratesHybridTHC: 77.66%CBD: 0.15%" +``` + +### 4.4 Compute Discount Percentage + +```typescript +function computeDiscountPercent( + salePriceCents: number | null, + regularPriceCents: number | null +): number | null { + if (!salePriceCents || !regularPriceCents || regularPriceCents <= 0) { + return null; + } + + if (salePriceCents >= regularPriceCents) { + return null; // No discount + } + + return Math.round((1 - salePriceCents / regularPriceCents) * 100); +} +``` + +### 4.5 Updated Specials SQL Query + +Replace the specials query in Section 3.2 with this API-layer detection query: + +```sql +-- Get specials with API-layer detection +WITH detected_specials AS ( + SELECT + p.*, + -- Detect special using multiple rules + CASE + WHEN p.name ILIKE '%Special Offer%' THEN TRUE + WHEN p.sale_price IS NOT NULL + AND p.regular_price IS NOT NULL + AND p.sale_price::numeric < p.regular_price::numeric THEN TRUE + WHEN p.price IS NOT NULL + AND p.original_price IS NOT NULL + AND p.price::numeric < p.original_price::numeric THEN TRUE + WHEN p.special_text IS NOT NULL AND p.special_text != '' THEN TRUE + ELSE FALSE + END AS is_on_special, + + -- Determine special type + CASE + WHEN p.name ILIKE '%Special Offer%' THEN 'special_offer' + WHEN p.sale_price IS NOT NULL + AND p.regular_price IS NOT NULL + AND p.sale_price::numeric < p.regular_price::numeric THEN 'percent_off' + WHEN p.price IS NOT NULL + AND p.original_price IS NOT NULL + AND p.price::numeric < p.original_price::numeric THEN 'percent_off' + WHEN p.special_text IS NOT NULL AND p.special_text != '' THEN 'promo' + ELSE NULL + END AS computed_special_type, + + -- Compute discount percentage + CASE + WHEN p.sale_price IS NOT NULL + AND p.regular_price IS NOT NULL + AND p.regular_price::numeric > 0 + THEN ROUND((1 - p.sale_price::numeric / p.regular_price::numeric) * 100) + WHEN p.price IS NOT NULL + AND p.original_price IS NOT NULL + AND p.original_price::numeric > 0 + THEN ROUND((1 - p.price::numeric / p.original_price::numeric) * 100) + ELSE NULL + END AS discount_percent, + + -- Clean product name (remove "Special Offer" suffix) + REGEXP_REPLACE(p.name, '\s*Special Offer\s*$', '', 'i') AS clean_name + + FROM products p + WHERE p.store_id = $1 + AND p.in_stock = TRUE +) +SELECT + ds.id AS product_id, + ds.slug, + ds.clean_name AS name, + COALESCE(ds.brand || ' ' || ds.clean_name, ds.clean_name) AS full_name, + ds.brand AS brand_name, + cb.id AS brand_id, + c.id AS category_id, + c.name AS category_name, + c.slug AS category_slug, + ds.strain_type, + ds.weight, + ds.thc_percentage AS thc_percent, + ds.cbd_percentage AS cbd_percent, + ds.in_stock, + -- Price in cents + ROUND(COALESCE(ds.sale_price, ds.price, 0) * 100)::INTEGER AS price_cents, + -- Regular price in cents + ROUND(COALESCE(ds.regular_price, ds.original_price, ds.price) * 100)::INTEGER AS regular_price_cents, + ds.is_on_special, + ds.computed_special_type, + ds.discount_percent, + ds.image_url_full, + ds.image_url, + ds.thumbnail_path, + ds.medium_path, + ds.description, + ds.metadata, + ds.dutchie_url, + ds.last_seen_at AS last_updated_at, + -- Calculated savings + CASE + WHEN ds.regular_price IS NOT NULL AND ds.sale_price IS NOT NULL + THEN ROUND((ds.regular_price - ds.sale_price) * 100)::INTEGER + WHEN ds.original_price IS NOT NULL AND ds.price IS NOT NULL + THEN ROUND((ds.original_price - ds.price) * 100)::INTEGER + ELSE NULL + END AS savings_cents +FROM detected_specials ds +LEFT JOIN categories c ON c.id = ds.category_id +LEFT JOIN canonical_brands cb ON LOWER(cb.name) = LOWER(ds.brand) +WHERE ds.is_on_special = TRUE + AND ($2::TEXT IS NULL OR c.slug = $2) -- category filter + AND ($3::TEXT IS NULL OR ds.brand ILIKE '%' || $3 || '%') -- brand filter +ORDER BY + ds.discount_percent DESC NULLS LAST, + ds.name ASC +LIMIT $4 OFFSET $5; +``` + +### 4.6 Special Badge Generation + +Generate the badge text based on detected special type: + +```typescript +function generateSpecialBadge( + specialType: string | null, + discountPercent: number | null, + hasSpecialOfferInName: boolean +): string { + if (discountPercent && discountPercent > 0) { + return `${discountPercent}% OFF`; + } + + if (hasSpecialOfferInName) { + return 'SPECIAL'; + } + + switch (specialType) { + case 'percent_off': + return discountPercent ? `${discountPercent}% OFF` : 'SALE'; + case 'special_offer': + return 'SPECIAL'; + case 'promo': + return 'PROMO'; + default: + return 'DEAL'; + } +} +``` + +--- + +## 5. Special Text Parsing + +### 5.1 Parse Function + +The special parsing logic determines `special.type`, `special.badge_text`, and `special.value`: + +```typescript +interface ParsedSpecial { + type: 'percent_off' | 'dollar_off' | 'bogo' | 'bundle' | 'set_price' | 'other'; + text: string; // Raw text + badge_text: string; // Short display text + value: number | null; // Numeric value + savings_cents: number | null; // Calculated savings +} + +function parseSpecialText( + specialText: string | null, + currentPriceCents: number, + regularPriceCents: number | null +): ParsedSpecial | null { + if (!specialText || specialText.trim() === '') { + return null; + } + + const text = specialText.trim(); + const lower = text.toLowerCase(); + + // Calculate savings from price difference + const savingsCents = regularPriceCents && regularPriceCents > currentPriceCents + ? regularPriceCents - currentPriceCents + : null; + + // 1. Percent off: "20% off", "25% OFF", "Save 30%" + const pctMatch = lower.match(/(\d+(?:\.\d+)?)\s*%\s*(?:off|discount)?/i) + || lower.match(/save\s*(\d+(?:\.\d+)?)\s*%/i); + if (pctMatch) { + const percent = parseFloat(pctMatch[1]); + return { + type: 'percent_off', + text, + badge_text: `${Math.round(percent)}% OFF`, + value: percent, + savings_cents: savingsCents + }; + } + + // 2. Dollar off: "$5 off", "$10 OFF" + const dollarMatch = lower.match(/\$(\d+(?:\.\d+)?)\s*off/i); + if (dollarMatch) { + const dollars = parseFloat(dollarMatch[1]); + return { + type: 'dollar_off', + text, + badge_text: `$${Math.round(dollars)} OFF`, + value: dollars, + savings_cents: dollars * 100 + }; + } + + // 3. BOGO: "BOGO", "Buy One Get One", "B1G1", "Buy 2 Get 1" + if (/\bbogo\b|buy\s*one\s*get\s*one|b1g1/i.test(lower)) { + return { + type: 'bogo', + text, + badge_text: 'BOGO', + value: 50, // 50% effective discount + savings_cents: savingsCents + }; + } + + const bogoMatch = lower.match(/buy\s*(\d+)\s*get\s*(\d+)/i); + if (bogoMatch) { + const buy = parseInt(bogoMatch[1]); + const get = parseInt(bogoMatch[2]); + return { + type: 'bogo', + text, + badge_text: `B${buy}G${get}`, + value: Math.round((get / (buy + get)) * 100), + savings_cents: savingsCents + }; + } + + // 4. Bundle: "3 for $100", "2/$50" + const bundleMatch = lower.match(/(\d+)\s*(?:for|\/)\s*\$(\d+(?:\.\d+)?)/i); + if (bundleMatch) { + const qty = parseInt(bundleMatch[1]); + const price = parseFloat(bundleMatch[2]); + return { + type: 'bundle', + text, + badge_text: `${qty} FOR $${Math.round(price)}`, + value: price, + savings_cents: null // Can't calculate without knowing single item price + }; + } + + // 5. Set price: "Now $25", "Only $30" + const setPriceMatch = lower.match(/(?:now|only|just|sale)\s*\$(\d+(?:\.\d+)?)/i); + if (setPriceMatch) { + const setPrice = parseFloat(setPriceMatch[1]); + return { + type: 'set_price', + text, + badge_text: `NOW $${Math.round(setPrice)}`, + value: setPrice, + savings_cents: savingsCents + }; + } + + // 6. Other - has text but couldn't parse + return { + type: 'other', + text, + badge_text: 'SPECIAL', + value: null, + savings_cents: savingsCents + }; +} +``` + +--- + +## 5. Caching Strategy + +### 5.1 Server-Side Cache Headers + +All store API responses include cache control headers: + +```http +HTTP/1.1 200 OK +Content-Type: application/json +Cache-Control: public, max-age=300, stale-while-revalidate=60 +ETag: "abc123def456" +X-Cache-TTL: 300 +X-Generated-At: 2025-02-28T15:00:00Z +``` + +### 5.2 Recommended TTLs + +| Endpoint | TTL | Rationale | +|----------|-----|-----------| +| `/menu` | 5 minutes | Prices and stock change frequently | +| `/specials` | 5 minutes | Deals may be time-sensitive | +| `/products` | 5 minutes | Same as menu | +| `/categories` | 10 minutes | Category structure rarely changes | +| `/brands` | 10 minutes | Brand list rarely changes | +| `/product/:slug` | 5 minutes | Individual product details | + +### 5.3 WordPress Transient Strategy + +Recommended caching approach for the WP plugin: + +```php +class Cannabrands_Cache { + const PREFIX = 'cbm_'; + + // Default TTLs in seconds + const TTL_MENU = 300; // 5 minutes + const TTL_SPECIALS = 300; // 5 minutes + const TTL_CATEGORIES = 600; // 10 minutes + const TTL_BRANDS = 600; // 10 minutes + + /** + * Build a deterministic cache key + */ + public static function build_key(string $endpoint, array $params = []): string { + $store_key = get_option('cannabrands_store_key'); + + // Sort params for consistent key + ksort($params); + $param_hash = md5(serialize($params)); + + return self::PREFIX . $store_key . '_' . $endpoint . '_' . $param_hash; + } + + /** + * Get cached data or fetch from API + */ + public static function get_or_fetch( + string $endpoint, + array $params, + callable $fetch_callback, + int $ttl = self::TTL_MENU + ) { + $key = self::build_key($endpoint, $params); + + // Try cache first + $cached = get_transient($key); + if ($cached !== false) { + return $cached; + } + + // Fetch fresh data + $data = $fetch_callback(); + + if (!is_wp_error($data)) { + set_transient($key, $data, $ttl); + } + + return $data; + } + + /** + * Clear all plugin caches + */ + public static function clear_all(): void { + global $wpdb; + + $wpdb->query( + "DELETE FROM {$wpdb->options} + WHERE option_name LIKE '_transient_" . self::PREFIX . "%' + OR option_name LIKE '_transient_timeout_" . self::PREFIX . "%'" + ); + } +} +``` + +### 5.4 Cache Key Examples + +``` +cbm_deeply-rooted_menu_d41d8cd98f00b204e9800998ecf8427e +cbm_deeply-rooted_menu_7d793037a0760186574b0282f2f435e7 // With filters +cbm_deeply-rooted_specials_d41d8cd98f00b204e9800998ecf8427e +cbm_deeply-rooted_categories_d41d8cd98f00b204e9800998ecf8427e +``` + +--- + +## 6. Elementor Widget → API Mapping + +### 6.1 Menu Widget + +**Widget Controls:** + +| Control | Type | Default | Maps To | +|---------|------|---------|---------| +| Category | Select | All | `?category=` | +| Brand | Text | - | `?brand=` | +| Sort By | Select | Name A-Z | `?sort=` | +| Products Per Page | Number | 50 | `?per_page=` | +| In Stock Only | Toggle | Yes | `?in_stock=true` | +| Show Images | Toggle | Yes | - (frontend only) | +| Show Prices | Toggle | Yes | - (frontend only) | +| Show THC | Toggle | Yes | - (frontend only) | +| Show Special Badge | Toggle | Yes | - (frontend only) | +| Columns | Number | 4 | - (frontend only) | + +**API Call:** + +``` +GET /api/stores/{store_key}/menu + ?category={category} + &brand={brand} + &sort={sort} + &per_page={per_page} + &in_stock={in_stock} + &page=1 +``` + +**Fields Used from Response:** + +```typescript +// From ProductForStore +{ + product_id, // For data attributes + slug, // For URL generation + full_name, // Display name + brand_name, // Brand label + category, // Category badge + strain_type, // Strain indicator + weight, // Size label + thc_percent, // THC display + in_stock, // Stock badge + price_cents, // Price display + regular_price_cents, // Strikethrough price + is_on_special, // Special styling + special.badge_text, // Special badge + image_url, // Product image + dutchie_url // Link target +} +``` + +### 6.2 Specials Widget + +**Widget Controls:** + +| Control | Type | Default | Maps To | +|---------|------|---------|---------| +| Heading | Text | "Today's Specials" | - (frontend) | +| Category | Select | All | `?category=` | +| Brand | Text | - | `?brand=` | +| Deal Type | Select | All | `?special_type=` | +| Limit | Number | 8 | `?limit=` | +| Show Original Price | Toggle | Yes | - (frontend) | +| Show Savings | Toggle | Yes | - (frontend) | +| Layout | Select | Grid | - (frontend) | + +**API Call:** + +``` +GET /api/stores/{store_key}/specials + ?category={category} + &brand={brand} + &special_type={special_type} + &limit={limit} +``` + +**Fields Used:** + +```typescript +{ + product_id, + slug, + full_name, + brand_name, + category, + strain_type, + weight, + thc_percent, + price_cents, // Current price + regular_price_cents, // Strikethrough + special.type, // For styling + special.text, // Deal description + special.badge_text, // Badge + special.savings_cents, // Savings display + image_url, + dutchie_url +} +``` + +### 6.3 Carousel Widget + +**Widget Controls:** + +| Control | Type | Default | Maps To | +|---------|------|---------|---------| +| Heading | Text | "Featured Products" | - (frontend) | +| Source | Select | All Products | Endpoint selection | +| Category | Select | All | `?category=` | +| Specials Only | Toggle | No | `?specials_only=true` | +| Brand | Text | - | `?brand=` | +| Limit | Number | 12 | `?limit=` | +| Slides to Show | Number | 4 | - (frontend) | +| Autoplay | Toggle | No | - (frontend) | +| Autoplay Speed | Number | 3000 | - (frontend) | +| Show Arrows | Toggle | Yes | - (frontend) | +| Show Dots | Toggle | Yes | - (frontend) | + +**API Call:** + +``` +// If Specials Only = true: +GET /api/stores/{store_key}/specials + ?category={category} + &brand={brand} + &limit={limit} + +// Otherwise: +GET /api/stores/{store_key}/products + ?category={category} + &brand={brand} + &specials_only={specials_only} + &in_stock=true + &per_page={limit} +``` + +**Fields Used:** + +```typescript +{ + product_id, + slug, + full_name, + brand_name, + price_cents, + regular_price_cents, + is_on_special, + special.badge_text, + image_url, + dutchie_url +} +``` + +### 6.4 Category Navigation Widget + +**Widget Controls:** + +| Control | Type | Default | Maps To | +|---------|------|---------|---------| +| Layout | Select | Grid | - (frontend) | +| Show Product Count | Toggle | Yes | - (frontend) | +| Show Icons | Toggle | Yes | - (frontend) | +| Link Behavior | Select | Filter Menu | - (frontend) | + +**API Call:** + +``` +GET /api/stores/{store_key}/categories +``` + +**Fields Used:** + +```typescript +{ + id, + name, + slug, + icon, + product_count, + in_stock_count, + on_special_count, + children[] +} +``` + +--- + +## 7. Shortcode → API Mapping + +### 7.1 Menu Shortcode + +``` +[cannabrands_menu + category="flower" + brand="Raw Garden" + sort="price_asc" + limit="24" + columns="4" + show_images="yes" + show_price="yes" + show_thc="yes" + in_stock="yes" +] +``` + +**Maps to:** + +``` +GET /api/stores/{store_key}/menu + ?category=flower + &brand=Raw+Garden + &sort=price_asc + &per_page=24 + &in_stock=true +``` + +### 7.2 Specials Shortcode + +``` +[cannabrands_specials + category="pre-rolls" + brand="Thunder Bud" + type="percent_off" + limit="8" + layout="grid" + show_savings="yes" +] +``` + +**Maps to:** + +``` +GET /api/stores/{store_key}/specials + ?category=pre-rolls + &brand=Thunder+Bud + &special_type=percent_off + &limit=8 +``` + +### 7.3 Carousel Shortcode + +``` +[cannabrands_carousel + category="vapes" + specials_only="true" + limit="10" + visible="4" + autoplay="yes" + speed="3000" +] +``` + +**Maps to:** + +``` +GET /api/stores/{store_key}/specials + ?category=vapes + &limit=10 +``` + +### 7.4 Categories Shortcode + +``` +[cannabrands_categories + layout="grid" + columns="4" + show_count="yes" + show_icons="yes" +] +``` + +**Maps to:** + +``` +GET /api/stores/{store_key}/categories +``` + +### 7.5 Single Product Shortcode + +``` +[cannabrands_product slug="thunder-bud-1g-pre-roll" layout="card"] +``` + +**Maps to:** + +``` +GET /api/stores/{store_key}/product/thunder-bud-1g-pre-roll +``` + +--- + +## 8. Error Responses + +### 8.1 Standard Error Format + +```json +{ + "error": { + "code": "STORE_NOT_FOUND", + "message": "No store found with key: invalid-store", + "status": 404 + } +} +``` + +### 8.2 Error Codes + +| Code | HTTP Status | Description | +|------|-------------|-------------| +| `AUTH_REQUIRED` | 401 | Missing X-Store-API-Key header | +| `INVALID_API_KEY` | 401 | API key not found or inactive | +| `STORE_MISMATCH` | 403 | API key doesn't match requested store | +| `RATE_LIMITED` | 429 | Rate limit exceeded | +| `STORE_NOT_FOUND` | 404 | Store key not found | +| `PRODUCT_NOT_FOUND` | 404 | Product slug not found | +| `INVALID_PARAMS` | 400 | Invalid query parameters | +| `INTERNAL_ERROR` | 500 | Server error | + +### 8.3 Rate Limit Response + +```json +{ + "error": { + "code": "RATE_LIMITED", + "message": "Rate limit exceeded. Try again in 42 seconds.", + "status": 429, + "retry_after": 42 + } +} +``` + +**Headers:** + +```http +HTTP/1.1 429 Too Many Requests +Retry-After: 42 +X-RateLimit-Limit: 1000 +X-RateLimit-Remaining: 0 +X-RateLimit-Reset: 1709139600 +``` + +--- + +## 9. Data Source Summary + +### 9.1 Tables Used by Store API + +| Table | Used For | +|-------|----------| +| `stores` | Store info (name, slug, location) | +| `products` | Product data (name, price, THC, images, etc.) | +| `categories` | Category tree and counts | +| `canonical_brands` | Brand ID lookups (optional) | +| `store_api_keys` | Authentication | + +### 9.2 Tables NOT Used by Store API + +| Table | Belongs To | +|-------|------------| +| `brand_daily_metrics` | Brand Intelligence only | +| `brand_promo_daily_metrics` | Brand Intelligence only | +| `brand_store_events` | Brand Intelligence only | +| `brand_store_presence` | Brand Intelligence only | +| `store_products` | Brand Intelligence only | +| `store_product_snapshots` | Brand Intelligence only | +| `crawl_runs` | Internal crawler only | + +### 9.3 Data Flow Diagram + +``` +┌─────────────────────────────────────────────────────────────┐ +│ STORE API REQUEST │ +│ GET /api/stores/deeply-rooted/menu │ +└────────────────────────────┬────────────────────────────────┘ + │ + ▼ +┌─────────────────────────────────────────────────────────────┐ +│ AUTH MIDDLEWARE │ +│ • Validate X-Store-API-Key │ +│ • Check store_api_keys table │ +│ • Verify store_key matches │ +│ • Check rate limits │ +└────────────────────────────┬────────────────────────────────┘ + │ + ▼ +┌─────────────────────────────────────────────────────────────┐ +│ QUERY BUILDER │ +│ • Build SQL from query params │ +│ • Apply filters (category, brand, search, in_stock) │ +│ • Apply sorting │ +│ • Apply pagination │ +└────────────────────────────┬────────────────────────────────┘ + │ + ▼ +┌─────────────────────────────────────────────────────────────┐ +│ DATABASE QUERY │ +│ │ +│ SELECT FROM: │ +│ • products (main data) │ +│ • categories (category info) │ +│ • canonical_brands (brand_id lookup) │ +│ │ +│ NOT FROM: │ +│ • brand_daily_metrics │ +│ • brand_store_events │ +│ • store_product_snapshots │ +└────────────────────────────┬────────────────────────────────┘ + │ + ▼ +┌─────────────────────────────────────────────────────────────┐ +│ RESPONSE BUILDER │ +│ • Transform to ProductForStore format │ +│ • Parse special text │ +│ • Build image URLs │ +│ • Add cache headers │ +└────────────────────────────┬────────────────────────────────┘ + │ + ▼ +┌─────────────────────────────────────────────────────────────┐ +│ JSON RESPONSE │ +│ { store: {...}, meta: {...}, products: [...] } │ +└─────────────────────────────────────────────────────────────┘ +``` + +--- + +## Document Version + +| Version | Date | Author | Changes | +|---------|------|--------|---------| +| 1.0 | 2025-02-28 | Claude | Initial Store API specification | +| 1.1 | 2025-11-30 | Claude | Added API-layer specials detection (Section 4). Crawler is frozen - all specials detection now happens at query time. | diff --git a/docs/WORDPRESS_PLUGIN_SPEC.md b/docs/WORDPRESS_PLUGIN_SPEC.md new file mode 100644 index 00000000..64f925c7 --- /dev/null +++ b/docs/WORDPRESS_PLUGIN_SPEC.md @@ -0,0 +1,2994 @@ +# Cannabrands WordPress Menu Plugin - Technical Specification + +## Overview + +This document specifies the WordPress plugin that allows dispensaries to display their product menus, specials, and carousels on their WordPress websites - replacing the need for Dutchie Plus. + +### Mental Model + +``` +┌─────────────────────────────────────────────────────────────────────┐ +│ CRAWLER + CANONICAL TABLES │ +│ (Single Source of Truth) │ +└─────────────────────────────────┬───────────────────────────────────┘ + │ + ┌───────────────────┴───────────────────┐ + │ │ + ▼ ▼ +┌─────────────────────────────┐ ┌─────────────────────────────┐ +│ BRAND INTELLIGENCE │ │ STORE MENU PLUGIN │ +│ (Cannabrands App) │ │ (WordPress/Dispensaries) │ +├─────────────────────────────┤ ├─────────────────────────────┤ +│ • brand_daily_metrics │ │ • products (current state) │ +│ • brand_promo_daily_metrics │ │ • categories │ +│ • brand_store_events │ │ • specials (parsed deals) │ +│ • v_brand_store_detail │ │ │ +├─────────────────────────────┤ ├─────────────────────────────┤ +│ Auth: JWT with brand_key │ │ Auth: API key + store_id │ +│ Path: /v1/brands/:key/... │ │ Path: /v1/stores/:key/... │ +└─────────────────────────────┘ └─────────────────────────────┘ +``` + +### Non-Negotiable Constraints + +1. **DO NOT** modify the crawler +2. **DO NOT** break existing API endpoints +3. **DO NOT** couple plugin to intelligence/analytics tables +4. Store menu endpoints use **current state only** (not historical metrics) +5. Plugin is **read-only** - no writes to our database + +--- + +## 1. Store-Facing API Endpoints + +These endpoints are specifically for the WordPress plugin. They are separate from Brand Intelligence endpoints. + +### 1.1 Authentication + +**Header:** `X-Store-API-Key: {api_key}` + +API keys are issued per-store and stored in a `store_api_keys` table: + +```sql +CREATE TABLE store_api_keys ( + id SERIAL PRIMARY KEY, + store_id INTEGER NOT NULL REFERENCES stores(id), + api_key VARCHAR(64) NOT NULL, + name VARCHAR(100), -- "WordPress Plugin", "Mobile App", etc. + is_active BOOLEAN NOT NULL DEFAULT TRUE, + rate_limit INTEGER DEFAULT 1000, -- requests per hour + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + last_used_at TIMESTAMPTZ, + + CONSTRAINT uq_store_api_keys_key UNIQUE (api_key) +); + +CREATE INDEX idx_store_api_keys_key ON store_api_keys(api_key) WHERE is_active = TRUE; +CREATE INDEX idx_store_api_keys_store ON store_api_keys(store_id); +``` + +### 1.2 GET /v1/stores/:store_key/menu + +Returns the full product menu for a store, optionally grouped by category. + +**Path Parameters:** +| Param | Type | Description | +|-------|------|-------------| +| `store_key` | string | Store slug or ID | + +**Query Parameters:** +| Param | Type | Required | Default | Description | +|-------|------|----------|---------|-------------| +| `category` | string | No | - | Filter by category slug | +| `brand` | string | No | - | Filter by brand name | +| `in_stock` | boolean | No | `true` | Filter by stock status | +| `sort` | string | No | `name` | Sort: `name`, `price_asc`, `price_desc`, `thc_desc`, `newest` | +| `limit` | number | No | `500` | Max products (max 1000) | +| `offset` | number | No | `0` | Pagination offset | +| `group_by_category` | boolean | No | `false` | Group products by category | + +**Response (Flat List - `group_by_category=false`):** + +```json +{ + "store": { + "id": 1, + "name": "Deeply Rooted", + "slug": "deeply-rooted-sacramento", + "city": "Sacramento", + "state": "CA", + "logo_url": "https://...", + "dutchie_url": "https://dutchie.com/dispensary/deeply-rooted" + }, + "meta": { + "total": 342, + "limit": 50, + "offset": 0, + "has_more": true, + "generated_at": "2025-01-15T08:00:00.000Z", + "cache_ttl_seconds": 300 + }, + "products": [ + { + "id": 12345, + "slug": "raw-garden-live-resin-slurm-og-1g", + "name": "Live Resin - Slurm OG", + "full_name": "Raw Garden Live Resin - Slurm OG", + "brand": "Raw Garden", + "category": { + "id": 5, + "name": "Concentrates", + "slug": "concentrates", + "parent_slug": null + }, + "subcategory": { + "id": 12, + "name": "Live Resin", + "slug": "live-resin", + "parent_slug": "concentrates" + }, + "strain_type": "hybrid", + "weight": "1g", + "thc_percent": 82.5, + "cbd_percent": 0.1, + "price": { + "current": 45.00, + "regular": 55.00, + "is_on_special": true, + "discount_percent": 18.2, + "formatted": { + "current": "$45.00", + "regular": "$55.00", + "savings": "$10.00" + } + }, + "special": { + "active": true, + "text": "20% off all Raw Garden", + "type": "percent_off", + "ends_at": null + }, + "stock": { + "in_stock": true, + "quantity": null + }, + "images": { + "thumbnail": "https://images.dutchie.com/.../thumb.jpg", + "medium": "https://images.dutchie.com/.../medium.jpg", + "full": "https://images.dutchie.com/.../full.jpg" + }, + "description": "Premium live resin with...", + "terpenes": ["Limonene", "Myrcene", "Caryophyllene"], + "effects": ["Relaxed", "Happy", "Euphoric"], + "dutchie_url": "https://dutchie.com/dispensary/deeply-rooted/product/raw-garden-live-resin-slurm-og", + "last_updated": "2025-01-15T06:30:00.000Z" + } + ] +} +``` + +**Response (Grouped - `group_by_category=true`):** + +```json +{ + "store": { ... }, + "meta": { + "total": 342, + "categories_count": 8, + "generated_at": "2025-01-15T08:00:00.000Z", + "cache_ttl_seconds": 300 + }, + "categories": [ + { + "id": 1, + "name": "Flower", + "slug": "flower", + "product_count": 85, + "products": [ + { ... }, + { ... } + ] + }, + { + "id": 2, + "name": "Pre-Rolls", + "slug": "pre-rolls", + "product_count": 42, + "products": [ ... ] + } + ] +} +``` + +### 1.3 GET /v1/stores/:store_key/specials + +Returns products currently on special/deal. + +**Query Parameters:** +| Param | Type | Required | Default | Description | +|-------|------|----------|---------|-------------| +| `category` | string | No | - | Filter by category slug | +| `brand` | string | No | - | Filter by brand name | +| `special_type` | string | No | - | Filter: `percent_off`, `dollar_off`, `bogo`, `bundle` | +| `sort` | string | No | `discount_desc` | Sort: `discount_desc`, `price_asc`, `name` | +| `limit` | number | No | `50` | Max products | + +**Response:** + +```json +{ + "store": { ... }, + "meta": { + "total_specials": 34, + "generated_at": "2025-01-15T08:00:00.000Z", + "cache_ttl_seconds": 300 + }, + "specials": [ + { + "id": 12345, + "slug": "raw-garden-live-resin-slurm-og-1g", + "name": "Live Resin - Slurm OG", + "brand": "Raw Garden", + "category": { + "name": "Concentrates", + "slug": "concentrates" + }, + "strain_type": "hybrid", + "weight": "1g", + "thc_percent": 82.5, + "price": { + "current": 45.00, + "regular": 55.00, + "discount_percent": 18.2, + "formatted": { + "current": "$45.00", + "regular": "$55.00", + "savings": "$10.00" + } + }, + "special": { + "text": "20% off all Raw Garden", + "type": "percent_off", + "value": 20, + "badge_text": "20% OFF", + "ends_at": null + }, + "images": { + "thumbnail": "https://...", + "medium": "https://..." + }, + "in_stock": true, + "dutchie_url": "https://..." + } + ], + "summary": { + "by_type": { + "percent_off": 18, + "dollar_off": 5, + "bogo": 8, + "bundle": 2, + "other": 1 + }, + "by_category": [ + { "name": "Flower", "count": 12 }, + { "name": "Concentrates", "count": 10 }, + { "name": "Edibles", "count": 8 } + ], + "avg_discount_percent": 22.5 + } +} +``` + +### 1.4 GET /v1/stores/:store_key/categories + +Returns the category tree for a store's menu. + +**Response:** + +```json +{ + "store": { + "id": 1, + "name": "Deeply Rooted", + "slug": "deeply-rooted-sacramento" + }, + "categories": [ + { + "id": 1, + "name": "Flower", + "slug": "flower", + "product_count": 85, + "in_stock_count": 72, + "icon": "flower", + "children": [ + { + "id": 10, + "name": "Indoor", + "slug": "indoor", + "product_count": 45, + "in_stock_count": 40 + }, + { + "id": 11, + "name": "Outdoor", + "slug": "outdoor", + "product_count": 25, + "in_stock_count": 20 + } + ] + }, + { + "id": 2, + "name": "Pre-Rolls", + "slug": "pre-rolls", + "product_count": 42, + "in_stock_count": 38, + "icon": "joint", + "children": [] + }, + { + "id": 3, + "name": "Vapes", + "slug": "vapes", + "product_count": 65, + "in_stock_count": 58, + "icon": "vape", + "children": [ + { + "id": 20, + "name": "Cartridges", + "slug": "cartridges", + "product_count": 40 + }, + { + "id": 21, + "name": "Disposables", + "slug": "disposables", + "product_count": 25 + } + ] + } + ] +} +``` + +### 1.5 GET /v1/stores/:store_key/brands + +Returns all brands available at the store. + +**Response:** + +```json +{ + "store": { ... }, + "brands": [ + { + "name": "Raw Garden", + "slug": "raw-garden", + "product_count": 24, + "in_stock_count": 21, + "on_special_count": 3, + "logo_url": null, + "categories": ["Concentrates", "Vapes"] + }, + { + "name": "Stiiizy", + "slug": "stiiizy", + "product_count": 18, + "in_stock_count": 15, + "on_special_count": 0, + "logo_url": null, + "categories": ["Vapes", "Flower"] + } + ] +} +``` + +### 1.6 GET /v1/stores/:store_key/product/:product_slug + +Returns a single product's full details. + +**Response:** + +```json +{ + "product": { + "id": 12345, + "slug": "raw-garden-live-resin-slurm-og-1g", + "name": "Live Resin - Slurm OG", + "full_name": "Raw Garden Live Resin - Slurm OG", + "brand": "Raw Garden", + "category": { + "id": 5, + "name": "Concentrates", + "slug": "concentrates" + }, + "subcategory": { + "id": 12, + "name": "Live Resin", + "slug": "live-resin" + }, + "strain_type": "hybrid", + "weight": "1g", + "thc_percent": 82.5, + "cbd_percent": 0.1, + "price": { + "current": 45.00, + "regular": 55.00, + "is_on_special": true, + "discount_percent": 18.2 + }, + "special": { + "active": true, + "text": "20% off all Raw Garden", + "type": "percent_off" + }, + "in_stock": true, + "images": { + "thumbnail": "https://...", + "medium": "https://...", + "full": "https://..." + }, + "description": "Premium live resin extracted from fresh frozen cannabis...", + "terpenes": ["Limonene", "Myrcene", "Caryophyllene"], + "effects": ["Relaxed", "Happy", "Euphoric"], + "flavors": ["Citrus", "Earthy", "Pine"], + "lineage": "Unknown lineage", + "dutchie_url": "https://...", + "last_updated": "2025-01-15T06:30:00.000Z" + }, + "related_products": [ + { ... }, + { ... } + ] +} +``` + +--- + +## 2. WordPress Plugin Architecture + +### 2.1 Directory Structure + +``` +cannabrands-menu/ +├── cannabrands-menu.php # Main plugin file +├── readme.txt # WordPress.org readme +├── uninstall.php # Cleanup on uninstall +│ +├── includes/ +│ ├── class-cannabrands-plugin.php # Main plugin class +│ ├── class-cannabrands-api-client.php # API communication +│ ├── class-cannabrands-cache.php # Transient caching +│ ├── class-cannabrands-settings.php # Admin settings page +│ ├── class-cannabrands-shortcodes.php # Shortcode handlers +│ ├── class-cannabrands-rest-proxy.php # Optional: local REST proxy +│ │ +│ ├── elementor/ +│ │ ├── class-cannabrands-elementor.php # Elementor integration +│ │ ├── widgets/ +│ │ │ ├── class-widget-menu.php # Menu widget +│ │ │ ├── class-widget-specials.php # Specials widget +│ │ │ ├── class-widget-carousel.php # Product carousel +│ │ │ ├── class-widget-categories.php # Category list/grid +│ │ │ └── class-widget-product-card.php # Single product card +│ │ └── controls/ +│ │ └── class-category-control.php # Custom category selector +│ │ +│ └── blocks/ +│ ├── class-cannabrands-blocks.php # Gutenberg blocks registration +│ ├── menu/ +│ │ ├── block.json +│ │ ├── edit.js +│ │ └── render.php +│ ├── specials/ +│ │ ├── block.json +│ │ ├── edit.js +│ │ └── render.php +│ └── carousel/ +│ ├── block.json +│ ├── edit.js +│ └── render.php +│ +├── assets/ +│ ├── css/ +│ │ ├── cannabrands-menu.css # Base styles +│ │ ├── cannabrands-menu-elementor.css # Elementor-specific +│ │ └── cannabrands-menu-admin.css # Admin styles +│ │ +│ ├── js/ +│ │ ├── cannabrands-menu.js # Frontend JS (carousel, etc.) +│ │ ├── cannabrands-admin.js # Admin JS +│ │ └── blocks/ # Gutenberg block JS +│ │ +│ └── images/ +│ └── placeholder.png # Product placeholder +│ +├── templates/ +│ ├── menu/ +│ │ ├── menu-grid.php # Grid layout template +│ │ ├── menu-list.php # List layout template +│ │ └── menu-grouped.php # Category-grouped template +│ │ +│ ├── specials/ +│ │ ├── specials-grid.php +│ │ └── specials-banner.php +│ │ +│ ├── carousel/ +│ │ └── carousel.php +│ │ +│ ├── partials/ +│ │ ├── product-card.php # Reusable product card +│ │ ├── product-card-compact.php # Compact variant +│ │ ├── price-display.php # Price with special handling +│ │ ├── category-pill.php # Category badge +│ │ └── special-badge.php # Deal badge +│ │ +│ └── admin/ +│ └── settings-page.php +│ +└── languages/ + └── cannabrands-menu.pot # Translation template +``` + +### 2.2 Main Plugin Class + +```php +load_dependencies(); + $this->init_components(); + $this->register_hooks(); + } + + private function load_dependencies() { + require_once CANNABRANDS_MENU_PATH . 'includes/class-cannabrands-api-client.php'; + require_once CANNABRANDS_MENU_PATH . 'includes/class-cannabrands-cache.php'; + require_once CANNABRANDS_MENU_PATH . 'includes/class-cannabrands-settings.php'; + require_once CANNABRANDS_MENU_PATH . 'includes/class-cannabrands-shortcodes.php'; + + // Elementor (if active) + if (did_action('elementor/loaded')) { + require_once CANNABRANDS_MENU_PATH . 'includes/elementor/class-cannabrands-elementor.php'; + } + + // Gutenberg blocks + require_once CANNABRANDS_MENU_PATH . 'includes/blocks/class-cannabrands-blocks.php'; + } + + private function init_components() { + $this->settings = new Cannabrands_Settings(); + $this->cache = new Cannabrands_Cache(); + $this->api = new Cannabrands_Api_Client($this->settings, $this->cache); + + new Cannabrands_Shortcodes($this->api); + + if (did_action('elementor/loaded')) { + new Cannabrands_Elementor($this->api); + } + + new Cannabrands_Blocks($this->api); + } + + private function register_hooks() { + add_action('wp_enqueue_scripts', [$this, 'enqueue_frontend_assets']); + add_action('admin_enqueue_scripts', [$this, 'enqueue_admin_assets']); + + // AJAX handlers for live preview in admin + add_action('wp_ajax_cannabrands_preview_menu', [$this, 'ajax_preview_menu']); + add_action('wp_ajax_cannabrands_clear_cache', [$this, 'ajax_clear_cache']); + } + + public function enqueue_frontend_assets() { + wp_enqueue_style( + 'cannabrands-menu', + CANNABRANDS_MENU_URL . 'assets/css/cannabrands-menu.css', + [], + CANNABRANDS_MENU_VERSION + ); + + wp_enqueue_script( + 'cannabrands-menu', + CANNABRANDS_MENU_URL . 'assets/js/cannabrands-menu.js', + ['jquery'], + CANNABRANDS_MENU_VERSION, + true + ); + + // Pass config to JS + wp_localize_script('cannabrands-menu', 'cannabrandsMenu', [ + 'ajaxUrl' => admin_url('admin-ajax.php'), + 'nonce' => wp_create_nonce('cannabrands_menu'), + 'storeSlug' => $this->settings->get('store_slug'), + ]); + } + + public function enqueue_admin_assets($hook) { + if ($hook !== 'settings_page_cannabrands-menu') { + return; + } + + wp_enqueue_style( + 'cannabrands-menu-admin', + CANNABRANDS_MENU_URL . 'assets/css/cannabrands-menu-admin.css', + [], + CANNABRANDS_MENU_VERSION + ); + + wp_enqueue_script( + 'cannabrands-menu-admin', + CANNABRANDS_MENU_URL . 'assets/js/cannabrands-admin.js', + ['jquery'], + CANNABRANDS_MENU_VERSION, + true + ); + } + + public function ajax_clear_cache() { + check_ajax_referer('cannabrands_admin', 'nonce'); + + if (!current_user_can('manage_options')) { + wp_send_json_error('Unauthorized'); + } + + $this->cache->clear_all(); + wp_send_json_success(['message' => 'Cache cleared successfully']); + } +} + +// Initialize +function cannabrands_menu() { + return Cannabrands_Menu_Plugin::instance(); +} + +add_action('plugins_loaded', 'cannabrands_menu'); +``` + +### 2.3 API Client Class + +```php +settings = $settings; + $this->cache = $cache; + } + + /** + * Get the full menu for the configured store + * + * @param array $params { + * @type string $category Category slug filter + * @type string $brand Brand name filter + * @type bool $in_stock Filter by stock status (default: true) + * @type string $sort Sort order: name, price_asc, price_desc, thc_desc, newest + * @type int $limit Max products (default: 500) + * @type int $offset Pagination offset + * @type bool $group_by_category Group products by category + * } + * @return array|WP_Error + */ + public function get_menu(array $params = []) { + $defaults = [ + 'in_stock' => true, + 'sort' => 'name', + 'limit' => 500, + 'offset' => 0, + 'group_by_category' => false, + ]; + + $params = wp_parse_args($params, $defaults); + + $cache_key = $this->build_cache_key('menu', $params); + $cached = $this->cache->get($cache_key); + + if ($cached !== false) { + return $cached; + } + + $response = $this->request('GET', '/menu', $params); + + if (!is_wp_error($response)) { + $this->cache->set($cache_key, $response, $this->get_cache_ttl()); + } + + return $response; + } + + /** + * Get products currently on special + * + * @param array $params { + * @type string $category Category slug filter + * @type string $brand Brand name filter + * @type string $special_type Filter: percent_off, dollar_off, bogo, bundle + * @type string $sort Sort: discount_desc, price_asc, name + * @type int $limit Max products (default: 50) + * } + * @return array|WP_Error + */ + public function get_specials(array $params = []) { + $defaults = [ + 'sort' => 'discount_desc', + 'limit' => 50, + ]; + + $params = wp_parse_args($params, $defaults); + + $cache_key = $this->build_cache_key('specials', $params); + $cached = $this->cache->get($cache_key); + + if ($cached !== false) { + return $cached; + } + + $response = $this->request('GET', '/specials', $params); + + if (!is_wp_error($response)) { + $this->cache->set($cache_key, $response, $this->get_cache_ttl()); + } + + return $response; + } + + /** + * Get category tree + * + * @return array|WP_Error + */ + public function get_categories() { + $cache_key = $this->build_cache_key('categories', []); + $cached = $this->cache->get($cache_key); + + if ($cached !== false) { + return $cached; + } + + $response = $this->request('GET', '/categories'); + + if (!is_wp_error($response)) { + // Categories change less frequently - cache longer + $this->cache->set($cache_key, $response, $this->get_cache_ttl() * 2); + } + + return $response; + } + + /** + * Get brands list + * + * @return array|WP_Error + */ + public function get_brands() { + $cache_key = $this->build_cache_key('brands', []); + $cached = $this->cache->get($cache_key); + + if ($cached !== false) { + return $cached; + } + + $response = $this->request('GET', '/brands'); + + if (!is_wp_error($response)) { + $this->cache->set($cache_key, $response, $this->get_cache_ttl() * 2); + } + + return $response; + } + + /** + * Get single product details + * + * @param string $product_slug + * @return array|WP_Error + */ + public function get_product(string $product_slug) { + $cache_key = $this->build_cache_key('product', ['slug' => $product_slug]); + $cached = $this->cache->get($cache_key); + + if ($cached !== false) { + return $cached; + } + + $response = $this->request('GET', '/product/' . sanitize_title($product_slug)); + + if (!is_wp_error($response)) { + $this->cache->set($cache_key, $response, $this->get_cache_ttl()); + } + + return $response; + } + + /** + * Make API request + */ + private function request(string $method, string $endpoint, array $params = []) { + $base_url = $this->settings->get('api_base_url'); + $store_key = $this->settings->get('store_key'); + $api_key = $this->settings->get('api_key'); + + if (empty($base_url) || empty($store_key) || empty($api_key)) { + return new WP_Error( + 'cannabrands_not_configured', + __('Cannabrands Menu plugin is not configured. Please enter your API credentials in Settings.', 'cannabrands-menu') + ); + } + + $url = trailingslashit($base_url) . 'v1/stores/' . $store_key . $endpoint; + + if (!empty($params) && $method === 'GET') { + $url = add_query_arg($params, $url); + } + + $args = [ + 'method' => $method, + 'timeout' => 30, + 'headers' => [ + 'X-Store-API-Key' => $api_key, + 'Accept' => 'application/json', + 'User-Agent' => 'Cannabrands-Menu-Plugin/' . CANNABRANDS_MENU_VERSION, + ], + ]; + + $response = wp_remote_request($url, $args); + + if (is_wp_error($response)) { + return $response; + } + + $code = wp_remote_retrieve_response_code($response); + $body = wp_remote_retrieve_body($response); + $data = json_decode($body, true); + + if ($code !== 200) { + $message = isset($data['message']) ? $data['message'] : 'API request failed'; + return new WP_Error('cannabrands_api_error', $message, ['status' => $code]); + } + + return $data; + } + + private function build_cache_key(string $endpoint, array $params): string { + $store_key = $this->settings->get('store_key'); + $param_hash = md5(serialize($params)); + return "cannabrands_{$store_key}_{$endpoint}_{$param_hash}"; + } + + private function get_cache_ttl(): int { + return (int) $this->settings->get('cache_ttl', 300); // Default 5 minutes + } +} +``` + +### 2.4 Cache Class + +```php +query( + "DELETE FROM {$wpdb->options} + WHERE option_name LIKE '_transient_cannabrands_%' + OR option_name LIKE '_transient_timeout_cannabrands_%'" + ); + + // If object cache, try to flush our group + if (wp_using_ext_object_cache() && function_exists('wp_cache_flush_group')) { + wp_cache_flush_group(self::CACHE_GROUP); + } + } + + /** + * Get or set - fetch from cache or execute callback + * + * @param string $key + * @param callable $callback + * @param int $ttl + * @return mixed + */ + public function remember(string $key, callable $callback, int $ttl = 300) { + $cached = $this->get($key); + + if ($cached !== false) { + return $cached; + } + + $value = $callback(); + + if (!is_wp_error($value)) { + $this->set($key, $value, $ttl); + } + + return $value; + } +} +``` + +### 2.5 Settings Class + +```php +settings = get_option(self::OPTION_NAME, $this->get_defaults()); + + add_action('admin_menu', [$this, 'add_menu_page']); + add_action('admin_init', [$this, 'register_settings']); + } + + private function get_defaults(): array { + return [ + 'api_base_url' => 'https://api.cannabrands.app', + 'store_key' => '', + 'api_key' => '', + 'cache_ttl' => 300, + 'default_image' => '', + 'enable_dutchie_links' => true, + 'price_currency' => 'USD', + 'price_position' => 'before', // before or after + ]; + } + + public function get(string $key, $default = null) { + return isset($this->settings[$key]) ? $this->settings[$key] : $default; + } + + public function add_menu_page() { + add_options_page( + __('Cannabrands Menu Settings', 'cannabrands-menu'), + __('Cannabrands Menu', 'cannabrands-menu'), + 'manage_options', + 'cannabrands-menu', + [$this, 'render_settings_page'] + ); + } + + public function register_settings() { + register_setting(self::OPTION_NAME, self::OPTION_NAME, [ + 'sanitize_callback' => [$this, 'sanitize_settings'], + ]); + + // API Settings Section + add_settings_section( + 'cannabrands_api_section', + __('API Configuration', 'cannabrands-menu'), + [$this, 'render_api_section'], + 'cannabrands-menu' + ); + + add_settings_field( + 'api_base_url', + __('API Base URL', 'cannabrands-menu'), + [$this, 'render_text_field'], + 'cannabrands-menu', + 'cannabrands_api_section', + ['field' => 'api_base_url', 'description' => 'Usually https://api.cannabrands.app'] + ); + + add_settings_field( + 'store_key', + __('Store ID / Slug', 'cannabrands-menu'), + [$this, 'render_text_field'], + 'cannabrands-menu', + 'cannabrands_api_section', + ['field' => 'store_key', 'description' => 'Your store identifier (e.g., deeply-rooted-sacramento)'] + ); + + add_settings_field( + 'api_key', + __('API Key', 'cannabrands-menu'), + [$this, 'render_password_field'], + 'cannabrands-menu', + 'cannabrands_api_section', + ['field' => 'api_key', 'description' => 'Your store API key'] + ); + + // Cache Settings Section + add_settings_section( + 'cannabrands_cache_section', + __('Cache Settings', 'cannabrands-menu'), + [$this, 'render_cache_section'], + 'cannabrands-menu' + ); + + add_settings_field( + 'cache_ttl', + __('Cache Duration (seconds)', 'cannabrands-menu'), + [$this, 'render_number_field'], + 'cannabrands-menu', + 'cannabrands_cache_section', + ['field' => 'cache_ttl', 'min' => 60, 'max' => 3600, 'description' => 'How long to cache menu data (300 = 5 minutes)'] + ); + + // Display Settings Section + add_settings_section( + 'cannabrands_display_section', + __('Display Settings', 'cannabrands-menu'), + null, + 'cannabrands-menu' + ); + + add_settings_field( + 'enable_dutchie_links', + __('Enable Dutchie Links', 'cannabrands-menu'), + [$this, 'render_checkbox_field'], + 'cannabrands-menu', + 'cannabrands_display_section', + ['field' => 'enable_dutchie_links', 'label' => 'Link products to Dutchie product pages'] + ); + } + + public function render_settings_page() { + include CANNABRANDS_MENU_PATH . 'templates/admin/settings-page.php'; + } + + public function render_api_section() { + echo '

' . __('Enter your Cannabrands API credentials. Contact support@cannabrands.app if you need API access.', 'cannabrands-menu') . '

'; + } + + public function render_cache_section() { + echo '

' . __('Menu data is cached to improve performance. Click "Clear Cache" to fetch fresh data.', 'cannabrands-menu') . '

'; + echo ''; + echo ''; + } + + public function render_text_field($args) { + $field = $args['field']; + $value = esc_attr($this->get($field, '')); + $description = isset($args['description']) ? $args['description'] : ''; + + echo ""; + if ($description) { + echo "

{$description}

"; + } + } + + public function render_password_field($args) { + $field = $args['field']; + $value = esc_attr($this->get($field, '')); + $description = isset($args['description']) ? $args['description'] : ''; + + echo ""; + if ($description) { + echo "

{$description}

"; + } + } + + public function render_number_field($args) { + $field = $args['field']; + $value = esc_attr($this->get($field, '')); + $min = isset($args['min']) ? $args['min'] : 0; + $max = isset($args['max']) ? $args['max'] : 9999; + $description = isset($args['description']) ? $args['description'] : ''; + + echo ""; + if ($description) { + echo "

{$description}

"; + } + } + + public function render_checkbox_field($args) { + $field = $args['field']; + $checked = $this->get($field) ? 'checked' : ''; + $label = isset($args['label']) ? $args['label'] : ''; + + echo ""; + } + + public function sanitize_settings($input) { + $sanitized = []; + + $sanitized['api_base_url'] = esc_url_raw($input['api_base_url'] ?? ''); + $sanitized['store_key'] = sanitize_text_field($input['store_key'] ?? ''); + $sanitized['api_key'] = sanitize_text_field($input['api_key'] ?? ''); + $sanitized['cache_ttl'] = absint($input['cache_ttl'] ?? 300); + $sanitized['enable_dutchie_links'] = isset($input['enable_dutchie_links']) ? 1 : 0; + + // Validate cache TTL range + if ($sanitized['cache_ttl'] < 60) $sanitized['cache_ttl'] = 60; + if ($sanitized['cache_ttl'] > 3600) $sanitized['cache_ttl'] = 3600; + + return $sanitized; + } +} +``` + +--- + +## 3. Shortcodes + +### 3.1 Shortcode Registration + +```php +api = $api; + + add_shortcode('cannabrands_menu', [$this, 'render_menu']); + add_shortcode('cannabrands_specials', [$this, 'render_specials']); + add_shortcode('cannabrands_carousel', [$this, 'render_carousel']); + add_shortcode('cannabrands_categories', [$this, 'render_categories']); + add_shortcode('cannabrands_product', [$this, 'render_product']); + } + + /** + * [cannabrands_menu] + * + * Attributes: + * category - Filter by category slug (e.g., "flower", "vapes") + * brand - Filter by brand name + * sort - Sort order: name, price_asc, price_desc, thc_desc, newest + * limit - Max products to show (default: 100) + * layout - Display: grid, list, grouped (default: grid) + * columns - Grid columns: 2, 3, 4, 5 (default: 4) + * show_price - Show price: yes/no (default: yes) + * show_thc - Show THC%: yes/no (default: yes) + * show_stock - Show stock status: yes/no (default: yes) + * in_stock - Only show in-stock: yes/no (default: yes) + */ + public function render_menu($atts): string { + $atts = shortcode_atts([ + 'category' => '', + 'brand' => '', + 'sort' => 'name', + 'limit' => 100, + 'layout' => 'grid', + 'columns' => 4, + 'show_price' => 'yes', + 'show_thc' => 'yes', + 'show_stock' => 'yes', + 'in_stock' => 'yes', + ], $atts, 'cannabrands_menu'); + + $params = [ + 'sort' => sanitize_text_field($atts['sort']), + 'limit' => absint($atts['limit']), + 'in_stock' => $atts['in_stock'] === 'yes', + 'group_by_category' => $atts['layout'] === 'grouped', + ]; + + if (!empty($atts['category'])) { + $params['category'] = sanitize_text_field($atts['category']); + } + + if (!empty($atts['brand'])) { + $params['brand'] = sanitize_text_field($atts['brand']); + } + + $data = $this->api->get_menu($params); + + if (is_wp_error($data)) { + return $this->render_error($data); + } + + $template = $atts['layout'] === 'grouped' ? 'menu-grouped' : 'menu-grid'; + + return $this->load_template("menu/{$template}", [ + 'data' => $data, + 'atts' => $atts, + ]); + } + + /** + * [cannabrands_specials] + * + * Attributes: + * category - Filter by category slug + * brand - Filter by brand name + * type - Special type: percent_off, dollar_off, bogo, bundle + * limit - Max products (default: 10) + * layout - Display: grid, list, banner (default: grid) + * columns - Grid columns (default: 4) + * show_savings - Show savings amount: yes/no (default: yes) + */ + public function render_specials($atts): string { + $atts = shortcode_atts([ + 'category' => '', + 'brand' => '', + 'type' => '', + 'limit' => 10, + 'layout' => 'grid', + 'columns' => 4, + 'show_savings' => 'yes', + ], $atts, 'cannabrands_specials'); + + $params = [ + 'limit' => absint($atts['limit']), + ]; + + if (!empty($atts['category'])) { + $params['category'] = sanitize_text_field($atts['category']); + } + + if (!empty($atts['brand'])) { + $params['brand'] = sanitize_text_field($atts['brand']); + } + + if (!empty($atts['type'])) { + $params['special_type'] = sanitize_text_field($atts['type']); + } + + $data = $this->api->get_specials($params); + + if (is_wp_error($data)) { + return $this->render_error($data); + } + + $template = $atts['layout'] === 'banner' ? 'specials-banner' : 'specials-grid'; + + return $this->load_template("specials/{$template}", [ + 'data' => $data, + 'atts' => $atts, + ]); + } + + /** + * [cannabrands_carousel] + * + * Attributes: + * category - Filter by category slug + * brand - Filter by brand name + * specials_only - Only show specials: yes/no (default: no) + * limit - Max products (default: 12) + * visible - Visible items at once (default: 4) + * autoplay - Enable autoplay: yes/no (default: no) + * speed - Autoplay speed in ms (default: 3000) + * show_arrows - Show nav arrows: yes/no (default: yes) + * show_dots - Show pagination dots: yes/no (default: yes) + */ + public function render_carousel($atts): string { + $atts = shortcode_atts([ + 'category' => '', + 'brand' => '', + 'specials_only' => 'no', + 'limit' => 12, + 'visible' => 4, + 'autoplay' => 'no', + 'speed' => 3000, + 'show_arrows' => 'yes', + 'show_dots' => 'yes', + ], $atts, 'cannabrands_carousel'); + + if ($atts['specials_only'] === 'yes') { + $params = ['limit' => absint($atts['limit'])]; + + if (!empty($atts['category'])) { + $params['category'] = sanitize_text_field($atts['category']); + } + + if (!empty($atts['brand'])) { + $params['brand'] = sanitize_text_field($atts['brand']); + } + + $data = $this->api->get_specials($params); + $products = isset($data['specials']) ? $data['specials'] : []; + } else { + $params = [ + 'limit' => absint($atts['limit']), + 'in_stock' => true, + ]; + + if (!empty($atts['category'])) { + $params['category'] = sanitize_text_field($atts['category']); + } + + if (!empty($atts['brand'])) { + $params['brand'] = sanitize_text_field($atts['brand']); + } + + $data = $this->api->get_menu($params); + $products = isset($data['products']) ? $data['products'] : []; + } + + if (is_wp_error($data)) { + return $this->render_error($data); + } + + return $this->load_template('carousel/carousel', [ + 'products' => $products, + 'atts' => $atts, + ]); + } + + /** + * [cannabrands_categories] + * + * Attributes: + * layout - Display: grid, list, dropdown (default: grid) + * columns - Grid columns (default: 4) + * show_count - Show product count: yes/no (default: yes) + * show_icons - Show category icons: yes/no (default: yes) + * link_to - Link behavior: page, filter, none (default: filter) + */ + public function render_categories($atts): string { + $atts = shortcode_atts([ + 'layout' => 'grid', + 'columns' => 4, + 'show_count' => 'yes', + 'show_icons' => 'yes', + 'link_to' => 'filter', + ], $atts, 'cannabrands_categories'); + + $data = $this->api->get_categories(); + + if (is_wp_error($data)) { + return $this->render_error($data); + } + + return $this->load_template('categories/categories-grid', [ + 'data' => $data, + 'atts' => $atts, + ]); + } + + /** + * [cannabrands_product slug="..."] + * + * Attributes: + * slug - Product slug (required) + * layout - Display: card, full, compact (default: card) + */ + public function render_product($atts): string { + $atts = shortcode_atts([ + 'slug' => '', + 'layout' => 'card', + ], $atts, 'cannabrands_product'); + + if (empty($atts['slug'])) { + return '

' . __('Product slug is required', 'cannabrands-menu') . '

'; + } + + $data = $this->api->get_product($atts['slug']); + + if (is_wp_error($data)) { + return $this->render_error($data); + } + + return $this->load_template('partials/product-card', [ + 'product' => $data['product'], + 'layout' => $atts['layout'], + ]); + } + + private function load_template(string $template, array $vars = []): string { + $template_path = CANNABRANDS_MENU_PATH . 'templates/' . $template . '.php'; + + // Allow theme override + $theme_template = locate_template('cannabrands-menu/' . $template . '.php'); + if ($theme_template) { + $template_path = $theme_template; + } + + if (!file_exists($template_path)) { + return ''; + } + + extract($vars); + + ob_start(); + include $template_path; + return ob_get_clean(); + } + + private function render_error(WP_Error $error): string { + if (current_user_can('manage_options')) { + return '
' . + 'Cannabrands Menu Error: ' . + esc_html($error->get_error_message()) . + '
'; + } + + return ''; + } +} +``` + +### 3.2 Shortcode Summary Table + +| Shortcode | Purpose | Key Attributes | +|-----------|---------|----------------| +| `[cannabrands_menu]` | Full product menu | `category`, `brand`, `sort`, `limit`, `layout`, `columns` | +| `[cannabrands_specials]` | Products on special | `category`, `brand`, `type`, `limit`, `layout` | +| `[cannabrands_carousel]` | Horizontal product slider | `category`, `specials_only`, `limit`, `visible`, `autoplay` | +| `[cannabrands_categories]` | Category list/grid | `layout`, `columns`, `show_count`, `link_to` | +| `[cannabrands_product]` | Single product card | `slug`, `layout` | + +### 3.3 Example Shortcode Usage + +```html + +[cannabrands_menu] + + +[cannabrands_menu category="flower" sort="thc_desc" limit="50"] + + +[cannabrands_menu category="pre-rolls" brand="Raw Garden" columns="3"] + + +[cannabrands_specials limit="10" show_savings="yes"] + + +[cannabrands_specials type="bogo" layout="banner"] + + +[cannabrands_carousel category="concentrates" limit="8" autoplay="yes" speed="4000"] + + +[cannabrands_carousel specials_only="yes" limit="6" visible="3"] + + +[cannabrands_categories layout="grid" columns="4" show_icons="yes"] + + +[cannabrands_product slug="raw-garden-live-resin-slurm-og-1g"] +``` + +--- + +## 4. Elementor Widgets + +### 4.1 Elementor Integration Class + +```php +api = $api; + + add_action('elementor/widgets/register', [$this, 'register_widgets']); + add_action('elementor/elements/categories_registered', [$this, 'register_category']); + add_action('elementor/editor/after_enqueue_styles', [$this, 'editor_styles']); + } + + public function register_category($elements_manager) { + $elements_manager->add_category('cannabrands', [ + 'title' => __('Cannabrands Menu', 'cannabrands-menu'), + 'icon' => 'fa fa-cannabis', + ]); + } + + public function register_widgets($widgets_manager) { + require_once CANNABRANDS_MENU_PATH . 'includes/elementor/widgets/class-widget-menu.php'; + require_once CANNABRANDS_MENU_PATH . 'includes/elementor/widgets/class-widget-specials.php'; + require_once CANNABRANDS_MENU_PATH . 'includes/elementor/widgets/class-widget-carousel.php'; + require_once CANNABRANDS_MENU_PATH . 'includes/elementor/widgets/class-widget-categories.php'; + + $widgets_manager->register(new Cannabrands_Widget_Menu($this->api)); + $widgets_manager->register(new Cannabrands_Widget_Specials($this->api)); + $widgets_manager->register(new Cannabrands_Widget_Carousel($this->api)); + $widgets_manager->register(new Cannabrands_Widget_Categories($this->api)); + } + + public function editor_styles() { + wp_enqueue_style( + 'cannabrands-elementor-editor', + CANNABRANDS_MENU_URL . 'assets/css/cannabrands-menu-elementor.css', + [], + CANNABRANDS_MENU_VERSION + ); + } +} +``` + +### 4.2 Menu Widget + +```php +api = $api ?? cannabrands_menu()->api; + } + + public function get_name() { + return 'cannabrands_menu'; + } + + public function get_title() { + return __('Product Menu', 'cannabrands-menu'); + } + + public function get_icon() { + return 'eicon-posts-grid'; + } + + public function get_categories() { + return ['cannabrands']; + } + + public function get_keywords() { + return ['menu', 'products', 'cannabis', 'dispensary']; + } + + protected function register_controls() { + + // ========== CONTENT TAB ========== + + $this->start_controls_section('section_content', [ + 'label' => __('Content', 'cannabrands-menu'), + 'tab' => Controls_Manager::TAB_CONTENT, + ]); + + // Category filter + $this->add_control('category', [ + 'label' => __('Category', 'cannabrands-menu'), + 'type' => Controls_Manager::SELECT, + 'default' => '', + 'options' => $this->get_category_options(), + 'description' => __('Filter products by category', 'cannabrands-menu'), + ]); + + // Brand filter + $this->add_control('brand', [ + 'label' => __('Brand', 'cannabrands-menu'), + 'type' => Controls_Manager::TEXT, + 'default' => '', + 'placeholder' => __('e.g., Raw Garden', 'cannabrands-menu'), + 'description' => __('Filter products by brand name', 'cannabrands-menu'), + ]); + + // Sort order + $this->add_control('sort', [ + 'label' => __('Sort By', 'cannabrands-menu'), + 'type' => Controls_Manager::SELECT, + 'default' => 'name', + 'options' => [ + 'name' => __('Name (A-Z)', 'cannabrands-menu'), + 'price_asc' => __('Price (Low to High)', 'cannabrands-menu'), + 'price_desc' => __('Price (High to Low)', 'cannabrands-menu'), + 'thc_desc' => __('THC (High to Low)', 'cannabrands-menu'), + 'newest' => __('Newest First', 'cannabrands-menu'), + ], + ]); + + // Limit + $this->add_control('limit', [ + 'label' => __('Products to Show', 'cannabrands-menu'), + 'type' => Controls_Manager::NUMBER, + 'default' => 50, + 'min' => 1, + 'max' => 500, + ]); + + // In stock only + $this->add_control('in_stock_only', [ + 'label' => __('In Stock Only', 'cannabrands-menu'), + 'type' => Controls_Manager::SWITCHER, + 'default' => 'yes', + ]); + + $this->end_controls_section(); + + // ========== LAYOUT SECTION ========== + + $this->start_controls_section('section_layout', [ + 'label' => __('Layout', 'cannabrands-menu'), + 'tab' => Controls_Manager::TAB_CONTENT, + ]); + + // Layout type + $this->add_control('layout', [ + 'label' => __('Layout', 'cannabrands-menu'), + 'type' => Controls_Manager::SELECT, + 'default' => 'grid', + 'options' => [ + 'grid' => __('Grid', 'cannabrands-menu'), + 'list' => __('List', 'cannabrands-menu'), + 'grouped' => __('Grouped by Category', 'cannabrands-menu'), + ], + ]); + + // Columns + $this->add_responsive_control('columns', [ + 'label' => __('Columns', 'cannabrands-menu'), + 'type' => Controls_Manager::SELECT, + 'default' => '4', + 'tablet_default' => '3', + 'mobile_default' => '2', + 'options' => [ + '1' => '1', + '2' => '2', + '3' => '3', + '4' => '4', + '5' => '5', + '6' => '6', + ], + 'condition' => [ + 'layout' => ['grid', 'grouped'], + ], + 'selectors' => [ + '{{WRAPPER}} .cannabrands-menu-grid' => 'grid-template-columns: repeat({{VALUE}}, 1fr);', + ], + ]); + + // Gap + $this->add_responsive_control('gap', [ + 'label' => __('Gap', 'cannabrands-menu'), + 'type' => Controls_Manager::SLIDER, + 'size_units' => ['px', 'em'], + 'default' => ['size' => 20, 'unit' => 'px'], + 'selectors' => [ + '{{WRAPPER}} .cannabrands-menu-grid' => 'gap: {{SIZE}}{{UNIT}};', + ], + ]); + + $this->end_controls_section(); + + // ========== CARD CONTENT SECTION ========== + + $this->start_controls_section('section_card_content', [ + 'label' => __('Card Content', 'cannabrands-menu'), + 'tab' => Controls_Manager::TAB_CONTENT, + ]); + + $this->add_control('show_image', [ + 'label' => __('Show Image', 'cannabrands-menu'), + 'type' => Controls_Manager::SWITCHER, + 'default' => 'yes', + ]); + + $this->add_control('show_brand', [ + 'label' => __('Show Brand', 'cannabrands-menu'), + 'type' => Controls_Manager::SWITCHER, + 'default' => 'yes', + ]); + + $this->add_control('show_category', [ + 'label' => __('Show Category', 'cannabrands-menu'), + 'type' => Controls_Manager::SWITCHER, + 'default' => 'yes', + ]); + + $this->add_control('show_price', [ + 'label' => __('Show Price', 'cannabrands-menu'), + 'type' => Controls_Manager::SWITCHER, + 'default' => 'yes', + ]); + + $this->add_control('show_thc', [ + 'label' => __('Show THC %', 'cannabrands-menu'), + 'type' => Controls_Manager::SWITCHER, + 'default' => 'yes', + ]); + + $this->add_control('show_strain_type', [ + 'label' => __('Show Strain Type', 'cannabrands-menu'), + 'type' => Controls_Manager::SWITCHER, + 'default' => 'yes', + ]); + + $this->add_control('show_stock_badge', [ + 'label' => __('Show Stock Badge', 'cannabrands-menu'), + 'type' => Controls_Manager::SWITCHER, + 'default' => 'yes', + ]); + + $this->add_control('show_special_badge', [ + 'label' => __('Show Special Badge', 'cannabrands-menu'), + 'type' => Controls_Manager::SWITCHER, + 'default' => 'yes', + ]); + + $this->end_controls_section(); + + // ========== STYLE TAB ========== + + $this->start_controls_section('section_card_style', [ + 'label' => __('Card Style', 'cannabrands-menu'), + 'tab' => Controls_Manager::TAB_STYLE, + ]); + + $this->add_control('card_background', [ + 'label' => __('Background Color', 'cannabrands-menu'), + 'type' => Controls_Manager::COLOR, + 'selectors' => [ + '{{WRAPPER}} .cannabrands-product-card' => 'background-color: {{VALUE}};', + ], + ]); + + $this->add_control('card_border_radius', [ + 'label' => __('Border Radius', 'cannabrands-menu'), + 'type' => Controls_Manager::DIMENSIONS, + 'size_units' => ['px', '%'], + 'selectors' => [ + '{{WRAPPER}} .cannabrands-product-card' => 'border-radius: {{TOP}}{{UNIT}} {{RIGHT}}{{UNIT}} {{BOTTOM}}{{UNIT}} {{LEFT}}{{UNIT}};', + ], + ]); + + $this->add_group_control(\Elementor\Group_Control_Box_Shadow::get_type(), [ + 'name' => 'card_shadow', + 'selector' => '{{WRAPPER}} .cannabrands-product-card', + ]); + + $this->end_controls_section(); + + // Typography section + $this->start_controls_section('section_typography', [ + 'label' => __('Typography', 'cannabrands-menu'), + 'tab' => Controls_Manager::TAB_STYLE, + ]); + + $this->add_group_control(Group_Control_Typography::get_type(), [ + 'name' => 'product_name_typography', + 'label' => __('Product Name', 'cannabrands-menu'), + 'selector' => '{{WRAPPER}} .cannabrands-product-name', + ]); + + $this->add_group_control(Group_Control_Typography::get_type(), [ + 'name' => 'price_typography', + 'label' => __('Price', 'cannabrands-menu'), + 'selector' => '{{WRAPPER}} .cannabrands-product-price', + ]); + + $this->end_controls_section(); + } + + protected function render() { + $settings = $this->get_settings_for_display(); + + $params = [ + 'sort' => $settings['sort'], + 'limit' => $settings['limit'], + 'in_stock' => $settings['in_stock_only'] === 'yes', + 'group_by_category' => $settings['layout'] === 'grouped', + ]; + + if (!empty($settings['category'])) { + $params['category'] = $settings['category']; + } + + if (!empty($settings['brand'])) { + $params['brand'] = $settings['brand']; + } + + $data = $this->api->get_menu($params); + + if (is_wp_error($data)) { + if (\Elementor\Plugin::$instance->editor->is_edit_mode()) { + echo '
' . esc_html($data->get_error_message()) . '
'; + } + return; + } + + $products = isset($data['products']) ? $data['products'] : []; + + if (empty($products) && \Elementor\Plugin::$instance->editor->is_edit_mode()) { + echo '
' . __('No products found. Adjust your filters or check API settings.', 'cannabrands-menu') . '
'; + return; + } + + include CANNABRANDS_MENU_PATH . 'includes/elementor/widgets/views/menu-view.php'; + } + + private function get_category_options(): array { + $options = ['' => __('All Categories', 'cannabrands-menu')]; + + $data = $this->api->get_categories(); + + if (!is_wp_error($data) && isset($data['categories'])) { + foreach ($data['categories'] as $cat) { + $options[$cat['slug']] = $cat['name']; + + // Add children with indentation + if (!empty($cat['children'])) { + foreach ($cat['children'] as $child) { + $options[$child['slug']] = '— ' . $child['name']; + } + } + } + } + + return $options; + } +} +``` + +### 4.3 Specials Widget + +```php +api = $api ?? cannabrands_menu()->api; + } + + public function get_name() { + return 'cannabrands_specials'; + } + + public function get_title() { + return __('Specials & Deals', 'cannabrands-menu'); + } + + public function get_icon() { + return 'eicon-price-table'; + } + + public function get_categories() { + return ['cannabrands']; + } + + protected function register_controls() { + + $this->start_controls_section('section_content', [ + 'label' => __('Content', 'cannabrands-menu'), + 'tab' => Controls_Manager::TAB_CONTENT, + ]); + + // Heading + $this->add_control('heading', [ + 'label' => __('Heading', 'cannabrands-menu'), + 'type' => Controls_Manager::TEXT, + 'default' => __("Today's Specials", 'cannabrands-menu'), + ]); + + // Category filter + $this->add_control('category', [ + 'label' => __('Category', 'cannabrands-menu'), + 'type' => Controls_Manager::SELECT, + 'default' => '', + 'options' => $this->get_category_options(), + ]); + + // Brand filter + $this->add_control('brand', [ + 'label' => __('Brand', 'cannabrands-menu'), + 'type' => Controls_Manager::TEXT, + 'default' => '', + ]); + + // Special type filter + $this->add_control('special_type', [ + 'label' => __('Deal Type', 'cannabrands-menu'), + 'type' => Controls_Manager::SELECT, + 'default' => '', + 'options' => [ + '' => __('All Types', 'cannabrands-menu'), + 'percent_off' => __('Percent Off', 'cannabrands-menu'), + 'dollar_off' => __('Dollar Off', 'cannabrands-menu'), + 'bogo' => __('BOGO', 'cannabrands-menu'), + 'bundle' => __('Bundle Deals', 'cannabrands-menu'), + ], + ]); + + // Limit + $this->add_control('limit', [ + 'label' => __('Products to Show', 'cannabrands-menu'), + 'type' => Controls_Manager::NUMBER, + 'default' => 8, + 'min' => 1, + 'max' => 50, + ]); + + $this->end_controls_section(); + + // Layout section + $this->start_controls_section('section_layout', [ + 'label' => __('Layout', 'cannabrands-menu'), + 'tab' => Controls_Manager::TAB_CONTENT, + ]); + + $this->add_control('layout', [ + 'label' => __('Layout', 'cannabrands-menu'), + 'type' => Controls_Manager::SELECT, + 'default' => 'grid', + 'options' => [ + 'grid' => __('Grid', 'cannabrands-menu'), + 'list' => __('List', 'cannabrands-menu'), + 'banner' => __('Banner Style', 'cannabrands-menu'), + ], + ]); + + $this->add_responsive_control('columns', [ + 'label' => __('Columns', 'cannabrands-menu'), + 'type' => Controls_Manager::SELECT, + 'default' => '4', + 'options' => ['1' => '1', '2' => '2', '3' => '3', '4' => '4', '5' => '5'], + 'condition' => ['layout' => 'grid'], + 'selectors' => [ + '{{WRAPPER}} .cannabrands-specials-grid' => 'grid-template-columns: repeat({{VALUE}}, 1fr);', + ], + ]); + + $this->end_controls_section(); + + // Display options + $this->start_controls_section('section_display', [ + 'label' => __('Display Options', 'cannabrands-menu'), + 'tab' => Controls_Manager::TAB_CONTENT, + ]); + + $this->add_control('show_original_price', [ + 'label' => __('Show Original Price (Strikethrough)', 'cannabrands-menu'), + 'type' => Controls_Manager::SWITCHER, + 'default' => 'yes', + ]); + + $this->add_control('show_savings', [ + 'label' => __('Show Savings Amount', 'cannabrands-menu'), + 'type' => Controls_Manager::SWITCHER, + 'default' => 'yes', + ]); + + $this->add_control('show_deal_badge', [ + 'label' => __('Show Deal Badge', 'cannabrands-menu'), + 'type' => Controls_Manager::SWITCHER, + 'default' => 'yes', + ]); + + $this->add_control('show_deal_text', [ + 'label' => __('Show Deal Description', 'cannabrands-menu'), + 'type' => Controls_Manager::SWITCHER, + 'default' => 'yes', + ]); + + $this->end_controls_section(); + + // Badge styling + $this->start_controls_section('section_badge_style', [ + 'label' => __('Deal Badge', 'cannabrands-menu'), + 'tab' => Controls_Manager::TAB_STYLE, + ]); + + $this->add_control('badge_background', [ + 'label' => __('Badge Background', 'cannabrands-menu'), + 'type' => Controls_Manager::COLOR, + 'default' => '#ef4444', + 'selectors' => [ + '{{WRAPPER}} .cannabrands-special-badge' => 'background-color: {{VALUE}};', + ], + ]); + + $this->add_control('badge_color', [ + 'label' => __('Badge Text Color', 'cannabrands-menu'), + 'type' => Controls_Manager::COLOR, + 'default' => '#ffffff', + 'selectors' => [ + '{{WRAPPER}} .cannabrands-special-badge' => 'color: {{VALUE}};', + ], + ]); + + $this->end_controls_section(); + } + + protected function render() { + $settings = $this->get_settings_for_display(); + + $params = [ + 'limit' => $settings['limit'], + ]; + + if (!empty($settings['category'])) { + $params['category'] = $settings['category']; + } + + if (!empty($settings['brand'])) { + $params['brand'] = $settings['brand']; + } + + if (!empty($settings['special_type'])) { + $params['special_type'] = $settings['special_type']; + } + + $data = $this->api->get_specials($params); + + if (is_wp_error($data)) { + if (\Elementor\Plugin::$instance->editor->is_edit_mode()) { + echo '
' . esc_html($data->get_error_message()) . '
'; + } + return; + } + + $specials = isset($data['specials']) ? $data['specials'] : []; + + include CANNABRANDS_MENU_PATH . 'includes/elementor/widgets/views/specials-view.php'; + } + + private function get_category_options(): array { + // Same implementation as Menu widget + return ['' => __('All Categories', 'cannabrands-menu')]; + } +} +``` + +### 4.4 Carousel Widget + +```php +api = $api ?? cannabrands_menu()->api; + } + + public function get_name() { + return 'cannabrands_carousel'; + } + + public function get_title() { + return __('Product Carousel', 'cannabrands-menu'); + } + + public function get_icon() { + return 'eicon-slider-push'; + } + + public function get_categories() { + return ['cannabrands']; + } + + public function get_script_depends() { + return ['cannabrands-carousel']; + } + + protected function register_controls() { + + // Content section + $this->start_controls_section('section_content', [ + 'label' => __('Content', 'cannabrands-menu'), + 'tab' => Controls_Manager::TAB_CONTENT, + ]); + + $this->add_control('heading', [ + 'label' => __('Heading', 'cannabrands-menu'), + 'type' => Controls_Manager::TEXT, + 'default' => __('Featured Products', 'cannabrands-menu'), + ]); + + $this->add_control('source', [ + 'label' => __('Product Source', 'cannabrands-menu'), + 'type' => Controls_Manager::SELECT, + 'default' => 'all', + 'options' => [ + 'all' => __('All Products', 'cannabrands-menu'), + 'specials' => __('Specials Only', 'cannabrands-menu'), + 'category' => __('Specific Category', 'cannabrands-menu'), + 'brand' => __('Specific Brand', 'cannabrands-menu'), + ], + ]); + + $this->add_control('category', [ + 'label' => __('Category', 'cannabrands-menu'), + 'type' => Controls_Manager::SELECT, + 'default' => '', + 'options' => $this->get_category_options(), + 'condition' => ['source' => ['category', 'all']], + ]); + + $this->add_control('brand', [ + 'label' => __('Brand', 'cannabrands-menu'), + 'type' => Controls_Manager::TEXT, + 'default' => '', + 'condition' => ['source' => ['brand', 'all']], + ]); + + $this->add_control('limit', [ + 'label' => __('Products to Load', 'cannabrands-menu'), + 'type' => Controls_Manager::NUMBER, + 'default' => 12, + 'min' => 4, + 'max' => 24, + ]); + + $this->end_controls_section(); + + // Carousel settings + $this->start_controls_section('section_carousel', [ + 'label' => __('Carousel Settings', 'cannabrands-menu'), + 'tab' => Controls_Manager::TAB_CONTENT, + ]); + + $this->add_responsive_control('slides_to_show', [ + 'label' => __('Slides to Show', 'cannabrands-menu'), + 'type' => Controls_Manager::NUMBER, + 'default' => 4, + 'tablet_default' => 3, + 'mobile_default' => 1, + 'min' => 1, + 'max' => 6, + ]); + + $this->add_control('autoplay', [ + 'label' => __('Autoplay', 'cannabrands-menu'), + 'type' => Controls_Manager::SWITCHER, + 'default' => '', + ]); + + $this->add_control('autoplay_speed', [ + 'label' => __('Autoplay Speed (ms)', 'cannabrands-menu'), + 'type' => Controls_Manager::NUMBER, + 'default' => 3000, + 'min' => 1000, + 'max' => 10000, + 'step' => 500, + 'condition' => ['autoplay' => 'yes'], + ]); + + $this->add_control('infinite', [ + 'label' => __('Infinite Loop', 'cannabrands-menu'), + 'type' => Controls_Manager::SWITCHER, + 'default' => 'yes', + ]); + + $this->add_control('show_arrows', [ + 'label' => __('Show Arrows', 'cannabrands-menu'), + 'type' => Controls_Manager::SWITCHER, + 'default' => 'yes', + ]); + + $this->add_control('show_dots', [ + 'label' => __('Show Dots', 'cannabrands-menu'), + 'type' => Controls_Manager::SWITCHER, + 'default' => 'yes', + ]); + + $this->end_controls_section(); + + // Card content options + $this->start_controls_section('section_card', [ + 'label' => __('Card Content', 'cannabrands-menu'), + 'tab' => Controls_Manager::TAB_CONTENT, + ]); + + $this->add_control('show_brand', [ + 'label' => __('Show Brand', 'cannabrands-menu'), + 'type' => Controls_Manager::SWITCHER, + 'default' => 'yes', + ]); + + $this->add_control('show_price', [ + 'label' => __('Show Price', 'cannabrands-menu'), + 'type' => Controls_Manager::SWITCHER, + 'default' => 'yes', + ]); + + $this->add_control('show_thc', [ + 'label' => __('Show THC', 'cannabrands-menu'), + 'type' => Controls_Manager::SWITCHER, + 'default' => '', + ]); + + $this->add_control('show_special_badge', [ + 'label' => __('Show Special Badge', 'cannabrands-menu'), + 'type' => Controls_Manager::SWITCHER, + 'default' => 'yes', + ]); + + $this->end_controls_section(); + + // Arrow styling + $this->start_controls_section('section_arrows_style', [ + 'label' => __('Navigation Arrows', 'cannabrands-menu'), + 'tab' => Controls_Manager::TAB_STYLE, + 'condition' => ['show_arrows' => 'yes'], + ]); + + $this->add_control('arrow_color', [ + 'label' => __('Arrow Color', 'cannabrands-menu'), + 'type' => Controls_Manager::COLOR, + 'selectors' => [ + '{{WRAPPER}} .cannabrands-carousel-arrow' => 'color: {{VALUE}};', + ], + ]); + + $this->add_control('arrow_background', [ + 'label' => __('Arrow Background', 'cannabrands-menu'), + 'type' => Controls_Manager::COLOR, + 'selectors' => [ + '{{WRAPPER}} .cannabrands-carousel-arrow' => 'background-color: {{VALUE}};', + ], + ]); + + $this->end_controls_section(); + } + + protected function render() { + $settings = $this->get_settings_for_display(); + + // Determine which API to call based on source + if ($settings['source'] === 'specials') { + $params = ['limit' => $settings['limit']]; + + if (!empty($settings['category'])) { + $params['category'] = $settings['category']; + } + + $data = $this->api->get_specials($params); + $products = isset($data['specials']) ? $data['specials'] : []; + } else { + $params = [ + 'limit' => $settings['limit'], + 'in_stock' => true, + ]; + + if (!empty($settings['category'])) { + $params['category'] = $settings['category']; + } + + if (!empty($settings['brand'])) { + $params['brand'] = $settings['brand']; + } + + $data = $this->api->get_menu($params); + $products = isset($data['products']) ? $data['products'] : []; + } + + if (is_wp_error($data)) { + if (\Elementor\Plugin::$instance->editor->is_edit_mode()) { + echo '
' . esc_html($data->get_error_message()) . '
'; + } + return; + } + + // Build carousel config for JS + $carousel_config = [ + 'slidesToShow' => (int) $settings['slides_to_show'], + 'slidesToShowTablet' => (int) ($settings['slides_to_show_tablet'] ?? 3), + 'slidesToShowMobile' => (int) ($settings['slides_to_show_mobile'] ?? 1), + 'autoplay' => $settings['autoplay'] === 'yes', + 'autoplaySpeed' => (int) $settings['autoplay_speed'], + 'infinite' => $settings['infinite'] === 'yes', + 'arrows' => $settings['show_arrows'] === 'yes', + 'dots' => $settings['show_dots'] === 'yes', + ]; + + include CANNABRANDS_MENU_PATH . 'includes/elementor/widgets/views/carousel-view.php'; + } + + private function get_category_options(): array { + return ['' => __('All Categories', 'cannabrands-menu')]; + } +} +``` + +### 4.5 Elementor Widget Summary + +| Widget | Controls | Data Source | Renders | +|--------|----------|-------------|---------| +| **Product Menu** | Category, Brand, Sort, Limit, Layout (grid/list/grouped), Columns, Card content toggles, Typography, Colors | `GET /menu` | Product grid/list with filtering | +| **Specials & Deals** | Category, Brand, Deal Type, Limit, Layout, Show savings/badges/deal text, Badge styling | `GET /specials` | Special products with deal badges | +| **Product Carousel** | Source (all/specials/category/brand), Slides to show, Autoplay, Speed, Arrows, Dots, Card content | `GET /menu` or `GET /specials` | Horizontal slider | +| **Category List** | Layout (grid/list/dropdown), Columns, Show count/icons, Link behavior | `GET /categories` | Category navigation | + +--- + +## 5. Caching Strategy + +### 5.1 Cache Layers + +``` +┌─────────────────────────────────────────────────────────────────┐ +│ REQUEST FLOW │ +└─────────────────────────────────────────────────────────────────┘ + │ + ▼ +┌─────────────────────────────────────────────────────────────────┐ +│ LAYER 1: Object Cache (Redis/Memcached if available) │ +│ TTL: 5 minutes │ +│ Key: cannabrands_{store}_{endpoint}_{params_hash} │ +└─────────────────────────────┬───────────────────────────────────┘ + │ MISS + ▼ +┌─────────────────────────────────────────────────────────────────┐ +│ LAYER 2: WordPress Transients (Database) │ +│ TTL: 5 minutes │ +│ Fallback when no object cache │ +└─────────────────────────────┬───────────────────────────────────┘ + │ MISS + ▼ +┌─────────────────────────────────────────────────────────────────┐ +│ LAYER 3: API Request │ +│ Result cached in Layer 1 or 2 │ +└─────────────────────────────────────────────────────────────────┘ +``` + +### 5.2 Cache Key Strategy + +```php +// Cache key format +"cannabrands_{store_key}_{endpoint}_{md5(serialized_params)}" + +// Examples: +"cannabrands_deeply-rooted_menu_a1b2c3d4" // Full menu +"cannabrands_deeply-rooted_menu_e5f6g7h8" // Menu with category filter +"cannabrands_deeply-rooted_specials_i9j0k1l2" // Specials +"cannabrands_deeply-rooted_categories_m3n4o5p6" // Categories (less volatile) +``` + +### 5.3 Cache TTL Recommendations + +| Endpoint | TTL | Rationale | +|----------|-----|-----------| +| `/menu` | 5 minutes | Products change frequently (price, stock) | +| `/specials` | 5 minutes | Deals may be time-sensitive | +| `/categories` | 10 minutes | Category structure changes rarely | +| `/brands` | 10 minutes | Brand list changes rarely | +| `/product/:slug` | 5 minutes | Individual product details | + +### 5.4 Cache Invalidation + +**Manual Invalidation:** +- Admin settings page "Clear Cache" button +- Clears all plugin transients + +**Automatic Invalidation:** +- Transients expire based on TTL +- No webhook-based invalidation (keep plugin simple) + +**Optional: Cron-based Prewarming:** +```php +// Schedule hourly cache refresh +add_action('cannabrands_cache_prewarm', function() { + $api = cannabrands_menu()->api; + + // Prewarm common requests + $api->get_menu(['limit' => 100]); + $api->get_specials(['limit' => 50]); + $api->get_categories(); +}); + +// Schedule the event +if (!wp_next_scheduled('cannabrands_cache_prewarm')) { + wp_schedule_event(time(), 'hourly', 'cannabrands_cache_prewarm'); +} +``` + +--- + +## 6. Decoupling from Intelligence Layer + +### 6.1 Separation of Concerns + +The WordPress plugin uses **current state data only**: + +| Plugin Uses | Plugin Does NOT Use | +|-------------|---------------------| +| `products` table (current) | `brand_daily_metrics` | +| `categories` table | `brand_promo_daily_metrics` | +| `stores` table | `brand_store_events` | +| `store_products` (current state) | `store_product_snapshots` (history) | +| Parsed special fields | Intelligence aggregations | + +### 6.2 API Path Separation + +``` +/v1/stores/:store_key/* → WordPress Plugin (store-scoped, current state) +/v1/brands/:brand_key/* → Cannabrands App (brand-scoped, analytics) +``` + +### 6.3 No Cross-Contamination + +The store-facing endpoints: +- Do NOT query `brand_daily_metrics` or any aggregation tables +- Do NOT expose brand identifiers or intelligence data +- Are read-only against `products` + `categories` + current `store_products` state +- Use simple JOIN queries, not analytical window functions + +### 6.4 Example Query (Store Menu) + +```sql +-- What the /stores/:key/menu endpoint queries: +SELECT + p.id, + p.slug, + p.name, + p.brand, + p.description, + p.thc_percentage, + p.cbd_percentage, + p.strain_type, + p.weight, + p.price, + p.original_price, + p.in_stock, + p.special_text, + p.image_url_full, + p.dutchie_url, + p.last_seen_at, + c.id as category_id, + c.name as category_name, + c.slug as category_slug +FROM products p +JOIN categories c ON c.id = p.category_id +WHERE p.store_id = $1 + AND p.in_stock = $2 -- optional filter + AND c.slug = $3 -- optional filter +ORDER BY p.name ASC +LIMIT $4 OFFSET $5; +``` + +No references to: +- `canonical_brands` +- `brand_store_presence` +- `brand_daily_metrics` +- `store_product_snapshots` + +--- + +## 7. Template Examples + +### 7.1 Product Card Template (`templates/partials/product-card.php`) + +```php + + +
+ + +
+ + + <?php echo esc_attr($product['name']); ?> + +
+ +
+ + + + + + + + + + + + + + +
+ + +
+ + +
+ +
+ + +

+ + + +

+ + +
+ +
+ + + +
+ THC: % + + CBD: % + +
+ + + +
+ + + + + + + + + + + + +
+ + + +
+ +
+ + +
+
+``` + +### 7.2 Base CSS (`assets/css/cannabrands-menu.css`) + +```css +/* ============================================ + CANNABRANDS MENU - BASE STYLES + ============================================ */ + +/* CSS Custom Properties for easy theming */ +:root { + --cannabrands-primary: #10b981; + --cannabrands-primary-dark: #059669; + --cannabrands-sale: #ef4444; + --cannabrands-indica: #6366f1; + --cannabrands-sativa: #f59e0b; + --cannabrands-hybrid: #10b981; + --cannabrands-text: #1f2937; + --cannabrands-text-muted: #6b7280; + --cannabrands-border: #e5e7eb; + --cannabrands-background: #ffffff; + --cannabrands-card-shadow: 0 1px 3px rgba(0, 0, 0, 0.1); + --cannabrands-card-radius: 8px; + --cannabrands-gap: 20px; +} + +/* Grid Layout */ +.cannabrands-menu-grid, +.cannabrands-specials-grid { + display: grid; + grid-template-columns: repeat(4, 1fr); + gap: var(--cannabrands-gap); +} + +@media (max-width: 1024px) { + .cannabrands-menu-grid, + .cannabrands-specials-grid { + grid-template-columns: repeat(3, 1fr); + } +} + +@media (max-width: 768px) { + .cannabrands-menu-grid, + .cannabrands-specials-grid { + grid-template-columns: repeat(2, 1fr); + } +} + +@media (max-width: 480px) { + .cannabrands-menu-grid, + .cannabrands-specials-grid { + grid-template-columns: 1fr; + } +} + +/* Product Card */ +.cannabrands-product-card { + background: var(--cannabrands-background); + border: 1px solid var(--cannabrands-border); + border-radius: var(--cannabrands-card-radius); + overflow: hidden; + transition: box-shadow 0.2s ease, transform 0.2s ease; +} + +.cannabrands-product-card:hover { + box-shadow: var(--cannabrands-card-shadow); + transform: translateY(-2px); +} + +/* Product Image */ +.cannabrands-product-image { + position: relative; + aspect-ratio: 1; + background: #f9fafb; +} + +.cannabrands-product-image img { + width: 100%; + height: 100%; + object-fit: cover; +} + +.cannabrands-product-placeholder { + display: flex; + align-items: center; + justify-content: center; + width: 100%; + height: 100%; + color: var(--cannabrands-text-muted); + font-size: 14px; +} + +/* Badges */ +.cannabrands-special-badge { + position: absolute; + top: 8px; + left: 8px; + background: var(--cannabrands-sale); + color: white; + font-size: 11px; + font-weight: 600; + padding: 4px 8px; + border-radius: 4px; + text-transform: uppercase; +} + +.cannabrands-strain-badge { + position: absolute; + top: 8px; + right: 8px; + font-size: 10px; + font-weight: 600; + padding: 3px 6px; + border-radius: 3px; + text-transform: uppercase; +} + +.cannabrands-strain-indica { background: var(--cannabrands-indica); color: white; } +.cannabrands-strain-sativa { background: var(--cannabrands-sativa); color: white; } +.cannabrands-strain-hybrid { background: var(--cannabrands-hybrid); color: white; } + +/* Product Info */ +.cannabrands-product-info { + padding: 12px; +} + +.cannabrands-product-brand { + font-size: 12px; + color: var(--cannabrands-text-muted); + margin-bottom: 4px; +} + +.cannabrands-product-name { + font-size: 14px; + font-weight: 600; + color: var(--cannabrands-text); + margin: 0 0 4px 0; + line-height: 1.3; +} + +.cannabrands-product-name a { + color: inherit; + text-decoration: none; +} + +.cannabrands-product-name a:hover { + color: var(--cannabrands-primary); +} + +.cannabrands-product-weight { + font-size: 12px; + color: var(--cannabrands-text-muted); + margin-bottom: 8px; +} + +.cannabrands-product-cannabinoids { + font-size: 12px; + color: var(--cannabrands-text-muted); + margin-bottom: 8px; +} + +.cannabrands-thc { + font-weight: 500; + color: var(--cannabrands-primary-dark); +} + +.cannabrands-cbd { + margin-left: 8px; +} + +/* Price */ +.cannabrands-product-price { + font-size: 16px; + font-weight: 700; +} + +.cannabrands-price-current { + color: var(--cannabrands-text); +} + +.cannabrands-price-sale { + color: var(--cannabrands-sale); +} + +.cannabrands-price-regular { + text-decoration: line-through; + color: var(--cannabrands-text-muted); + font-weight: 400; + font-size: 13px; + margin-right: 6px; +} + +/* Stock Status */ +.cannabrands-out-of-stock { + font-size: 11px; + color: var(--cannabrands-sale); + font-weight: 500; + margin-top: 6px; +} + +/* Carousel */ +.cannabrands-carousel { + position: relative; + overflow: hidden; +} + +.cannabrands-carousel-track { + display: flex; + transition: transform 0.3s ease; +} + +.cannabrands-carousel-slide { + flex: 0 0 25%; + padding: 0 10px; + box-sizing: border-box; +} + +.cannabrands-carousel-arrow { + position: absolute; + top: 50%; + transform: translateY(-50%); + width: 40px; + height: 40px; + background: var(--cannabrands-background); + border: 1px solid var(--cannabrands-border); + border-radius: 50%; + display: flex; + align-items: center; + justify-content: center; + cursor: pointer; + z-index: 10; + transition: background 0.2s; +} + +.cannabrands-carousel-arrow:hover { + background: #f3f4f6; +} + +.cannabrands-carousel-prev { left: 0; } +.cannabrands-carousel-next { right: 0; } + +.cannabrands-carousel-dots { + display: flex; + justify-content: center; + gap: 8px; + margin-top: 16px; +} + +.cannabrands-carousel-dot { + width: 8px; + height: 8px; + border-radius: 50%; + background: var(--cannabrands-border); + cursor: pointer; +} + +.cannabrands-carousel-dot.active { + background: var(--cannabrands-primary); +} + +/* Error States */ +.cannabrands-error { + padding: 20px; + background: #fef2f2; + border: 1px solid #fecaca; + border-radius: var(--cannabrands-card-radius); + color: #991b1b; + text-align: center; +} + +.cannabrands-empty { + padding: 40px 20px; + text-align: center; + color: var(--cannabrands-text-muted); +} +``` + +--- + +## 8. Implementation Checklist + +### Phase 1: API Endpoints +- [ ] Create `store_api_keys` table and migration +- [ ] Implement API key authentication middleware +- [ ] Implement `GET /v1/stores/:key/menu` endpoint +- [ ] Implement `GET /v1/stores/:key/specials` endpoint +- [ ] Implement `GET /v1/stores/:key/categories` endpoint +- [ ] Implement `GET /v1/stores/:key/brands` endpoint +- [ ] Implement `GET /v1/stores/:key/product/:slug` endpoint +- [ ] Add rate limiting +- [ ] Test with sample store + +### Phase 2: WordPress Plugin Core +- [ ] Set up plugin structure +- [ ] Implement Settings page +- [ ] Implement API Client class +- [ ] Implement Cache class +- [ ] Test API connection from WP + +### Phase 3: Shortcodes +- [ ] Implement `[cannabrands_menu]` +- [ ] Implement `[cannabrands_specials]` +- [ ] Implement `[cannabrands_carousel]` +- [ ] Implement `[cannabrands_categories]` +- [ ] Implement `[cannabrands_product]` +- [ ] Create template files +- [ ] Test all shortcodes + +### Phase 4: Elementor Integration +- [ ] Implement Menu widget +- [ ] Implement Specials widget +- [ ] Implement Carousel widget +- [ ] Implement Categories widget +- [ ] Test in Elementor editor +- [ ] Test live preview + +### Phase 5: Styling & Polish +- [ ] Complete CSS stylesheet +- [ ] Add responsive breakpoints +- [ ] Add dark mode support (optional) +- [ ] Add carousel JavaScript +- [ ] Browser testing +- [ ] Performance optimization + +### Phase 6: Documentation & Release +- [ ] Write user documentation +- [ ] Create readme.txt for WordPress.org +- [ ] Create screenshots +- [ ] Set up update mechanism +- [ ] Beta testing with pilot dispensary + +--- + +## Document Version + +| Version | Date | Author | Changes | +|---------|------|--------|---------| +| 1.0 | 2025-01-15 | Claude | Initial specification | diff --git a/frontend/src/pages/StoreDetail.tsx b/frontend/src/pages/StoreDetail.tsx index 74c45ec5..ca1dcde2 100644 --- a/frontend/src/pages/StoreDetail.tsx +++ b/frontend/src/pages/StoreDetail.tsx @@ -2,7 +2,10 @@ import { useEffect, useState } from 'react'; import { useParams, useNavigate } from 'react-router-dom'; import { Layout } from '../components/Layout'; import { api } from '../lib/api'; -import { Package, Tag, Zap, TrendingUp, Calendar, DollarSign } from 'lucide-react'; +import { + Package, Tag, Zap, Clock, ExternalLink, CheckCircle, XCircle, + AlertCircle, Building, MapPin, RefreshCw, Calendar, Activity +} from 'lucide-react'; export function StoreDetail() { const { slug } = useParams(); @@ -14,7 +17,7 @@ export function StoreDetail() { const [loading, setLoading] = useState(true); const [selectedCategory, setSelectedCategory] = useState(null); const [selectedBrand, setSelectedBrand] = useState(''); - const [view, setView] = useState<'products' | 'brands' | 'specials'>('products'); + const [view, setView] = useState<'products' | 'brands' | 'specials' | 'crawl-history'>('products'); const [sortBy, setSortBy] = useState('name'); useEffect(() => { @@ -30,19 +33,22 @@ export function StoreDetail() { const loadStoreData = async () => { setLoading(true); try { + // First, find store by slug to get its ID const allStores = await api.getStores(); - const storeData = allStores.stores.find((s: any) => s.slug === slug); + const basicStore = allStores.stores.find((s: any) => s.slug === slug); - if (!storeData) { + if (!basicStore) { throw new Error('Store not found'); } - const [categoriesData, brandsData] = await Promise.all([ - api.getCategories(storeData.id), - api.getStoreBrands(storeData.id) + // Fetch full store details using the enhanced endpoint + const [fullStoreData, categoriesData, brandsData] = await Promise.all([ + api.getStore(basicStore.id), + api.getCategories(basicStore.id), + api.getStoreBrands(basicStore.id) ]); - setStore(storeData); + setStore(fullStoreData); setCategories(categoriesData.categories || []); setBrands(brandsData.brands || []); } catch (error) { @@ -101,6 +107,43 @@ export function StoreDetail() { return 'https://via.placeholder.com/300x300?text=No+Image'; }; + const formatDate = (dateString: string | null) => { + if (!dateString) return 'Never'; + return new Date(dateString).toLocaleString('en-US', { + month: 'short', + day: 'numeric', + year: 'numeric', + hour: '2-digit', + minute: '2-digit' + }); + }; + + const getProviderBadgeColor = (provider: string) => { + switch (provider?.toLowerCase()) { + case 'dutchie': return 'bg-green-100 text-green-700'; + case 'jane': return 'bg-purple-100 text-purple-700'; + case 'treez': return 'bg-blue-100 text-blue-700'; + case 'weedmaps': return 'bg-orange-100 text-orange-700'; + case 'leafly': return 'bg-emerald-100 text-emerald-700'; + default: return 'bg-gray-100 text-gray-700'; + } + }; + + const getJobStatusBadge = (status: string) => { + switch (status) { + case 'completed': + return Completed; + case 'running': + return Running; + case 'failed': + return Failed; + case 'pending': + return Pending; + default: + return {status}; + } + }; + if (loading) { return ( @@ -127,33 +170,112 @@ export function StoreDetail() { return (
- {/* Header */} + {/* Header with Store Info */}
-
-
+
+
-

{store.name}

-

- {products.length} products • {categories.length} categories • {brands.length} brands -

+
+

{store.name}

+ + {store.provider || 'Unknown'} + +
+

Store ID: {store.id}

- View on Dutchie → + View Menu
+ {/* Stats Row */} +
+
+
+ + Products +
+

{store.product_count || 0}

+
+
+
+ + Categories +
+

{store.category_count || 0}

+
+
+
+ + In Stock +
+

{store.in_stock_count || 0}

+
+
+
+ + Out of Stock +
+

{store.out_of_stock_count || 0}

+
+
+
+ + Freshness +
+

+ {store.freshness || 'Never scraped'} +

+
+
+
+ + Next Crawl +
+

+ {store.schedule?.next_run_at ? formatDate(store.schedule.next_run_at) : 'Not scheduled'} +

+
+
+ + {/* Linked Dispensary */} + {store.linked_dispensary && ( +
+
+ + Linked Dispensary +
+
+
+

{store.linked_dispensary.name}

+

+ + {store.linked_dispensary.city}, {store.linked_dispensary.state} + {store.linked_dispensary.address && ` - ${store.linked_dispensary.address}`} +

+
+ +
+
+ )} + {/* View Tabs */}
+
+ {/* Crawl History View */} + {view === 'crawl-history' && ( +
+
+

Recent Crawl Jobs

+

Last 10 crawl jobs for this store

+
+ {store.recent_jobs && store.recent_jobs.length > 0 ? ( +
+ + + + + + + + + + + + + + + + + {store.recent_jobs.map((job: any) => ( + + + + + + + + + + + + + ))} + +
StatusTypeStartedCompletedFoundNewUpdatedIn StockOut of StockError
{getJobStatusBadge(job.status)}{job.job_type || '-'}{formatDate(job.started_at)}{formatDate(job.completed_at)}{job.products_found ?? '-'}{job.products_new ?? '-'}{job.products_updated ?? '-'}{job.in_stock_count ?? '-'}{job.out_of_stock_count ?? '-'} + {job.error_message || '-'} +
+
+ ) : ( +
+ +

No crawl history available

+
+ )} +
+ )} + {/* Products View */} {view === 'products' && ( <>