feat: SEO template library, discovery pipeline, and orchestrator enhancements

## SEO Template Library - Add complete template library with 7 page types (state, city, category, brand, product, search, regeneration) - Add Template Library tab in SEO Orchestrator with accordion-based editors - Add template preview, validation, and variable injection engine - Add API endpoints: /api/seo/templates, preview, validate, generate, regenerate ## Discovery Pipeline - Add promotion.ts for discovery location validation and promotion - Add discover-all-states.ts script for multi-state discovery - Add promotion log migration (067) - Enhance discovery routes and types ## Orchestrator & Admin - Add crawl_enabled filter to stores page - Add API permissions page - Add job queue management - Add price analytics routes - Add markets and intelligence routes - Enhance dashboard and worker monitoring ## Infrastructure - Add migrations for worker definitions, SEO settings, field alignment - Add canonical pipeline for scraper v2 - Update hydration and sync orchestrator - Enhance multi-state query service 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
2025-12-09 00:05:34 -07:00
parent 9711d594db
commit 2f483b3084
83 changed files with 16700 additions and 1277 deletions
--- a/backend/src/hydration/canonical-upsert.ts
+++ b/backend/src/hydration/canonical-upsert.ts
@@ -68,7 +68,7 @@ export async function upsertStoreProducts(
        const result = await client.query(
          `INSERT INTO store_products (
            dispensary_id, provider, provider_product_id, provider_brand_id,
-            name, brand_name, category, subcategory,
+            name_raw, brand_name_raw, category_raw, subcategory_raw,
            price_rec, price_med, price_rec_special, price_med_special,
            is_on_special, discount_percent,
            is_in_stock, stock_status,
@@ -87,10 +87,10 @@ export async function upsertStoreProducts(
          )
          ON CONFLICT (dispensary_id, provider, provider_product_id)
          DO UPDATE SET
-            name = EXCLUDED.name,
-            brand_name = EXCLUDED.brand_name,
-            category = EXCLUDED.category,
-            subcategory = EXCLUDED.subcategory,
+            name_raw = EXCLUDED.name_raw,
+            brand_name_raw = EXCLUDED.brand_name_raw,
+            category_raw = EXCLUDED.category_raw,
+            subcategory_raw = EXCLUDED.subcategory_raw,
            price_rec = EXCLUDED.price_rec,
            price_med = EXCLUDED.price_med,
            price_rec_special = EXCLUDED.price_rec_special,
@@ -122,8 +122,9 @@ export async function upsertStoreProducts(
            productPricing?.discountPercent,
            productAvailability?.inStock ?? true,
            productAvailability?.stockStatus || 'unknown',
-            product.thcPercent,
-            product.cbdPercent,
+            // Clamp THC/CBD to valid percentage range (0-100) - some products report mg as %
+            product.thcPercent !== null && product.thcPercent <= 100 ? product.thcPercent : null,
+            product.cbdPercent !== null && product.cbdPercent <= 100 ? product.cbdPercent : null,
            product.primaryImageUrl,
          ]
        );
@@ -212,8 +213,9 @@ export async function createStoreProductSnapshots(
        productAvailability?.inStock ?? true,
        productAvailability?.quantity,
        productAvailability?.stockStatus || 'unknown',
-        product.thcPercent,
-        product.cbdPercent,
+        // Clamp THC/CBD to valid percentage range (0-100) - some products report mg as %
+        product.thcPercent !== null && product.thcPercent <= 100 ? product.thcPercent : null,
+        product.cbdPercent !== null && product.cbdPercent <= 100 ? product.cbdPercent : null,
        product.primaryImageUrl,
        JSON.stringify(product.rawProduct),
      ]);
@@ -229,7 +231,7 @@ export async function createStoreProductSnapshots(
      `INSERT INTO store_product_snapshots (
        dispensary_id, provider, provider_product_id, crawl_run_id,
        captured_at,
-        name, brand_name, category, subcategory,
+        name_raw, brand_name_raw, category_raw, subcategory_raw,
        price_rec, price_med, price_rec_special, price_med_special,
        is_on_special, discount_percent,
        is_in_stock, stock_quantity, stock_status,
@@ -245,6 +247,202 @@ export async function createStoreProductSnapshots(
  return { created };
 }

+// ============================================================
+// VARIANT UPSERTS
+// ============================================================
+
+export interface UpsertVariantsResult {
+  upserted: number;
+  new: number;
+  updated: number;
+  snapshotsCreated: number;
+}
+
+/**
+ * Extract variant data from raw Dutchie product
+ */
+function extractVariantsFromRaw(rawProduct: any): any[] {
+  const children = rawProduct?.POSMetaData?.children || [];
+  return children.map((child: any) => ({
+    option: child.option || child.key || '',
+    canonicalSku: child.canonicalSKU || null,
+    canonicalId: child.canonicalID || null,
+    canonicalName: child.canonicalName || null,
+    priceRec: child.recPrice || child.price || null,
+    priceMed: child.medPrice || null,
+    priceRecSpecial: child.recSpecialPrice || null,
+    priceMedSpecial: child.medSpecialPrice || null,
+    quantity: child.quantityAvailable ?? child.quantity ?? null,
+    inStock: (child.quantityAvailable ?? child.quantity ?? 0) > 0,
+  }));
+}
+
+/**
+ * Parse weight value and unit from option string
+ * e.g., "1g" -> { value: 1, unit: "g" }
+ *       "3.5g" -> { value: 3.5, unit: "g" }
+ *       "1/8oz" -> { value: 0.125, unit: "oz" }
+ */
+function parseWeight(option: string): { value: number | null; unit: string | null } {
+  if (!option) return { value: null, unit: null };
+
+  // Handle fractions like "1/8oz"
+  const fractionMatch = option.match(/^(\d+)\/(\d+)\s*(g|oz|mg|ml)?$/i);
+  if (fractionMatch) {
+    const value = parseInt(fractionMatch[1]) / parseInt(fractionMatch[2]);
+    return { value, unit: fractionMatch[3]?.toLowerCase() || 'oz' };
+  }
+
+  // Handle decimals like "3.5g" or "100mg"
+  const decimalMatch = option.match(/^([\d.]+)\s*(g|oz|mg|ml|each)?$/i);
+  if (decimalMatch) {
+    return {
+      value: parseFloat(decimalMatch[1]),
+      unit: decimalMatch[2]?.toLowerCase() || null
+    };
+  }
+
+  return { value: null, unit: null };
+}
+
+/**
+ * Upsert variants for products and create variant snapshots
+ */
+export async function upsertProductVariants(
+  pool: Pool,
+  dispensaryId: number,
+  products: NormalizedProduct[],
+  crawlRunId: number | null,
+  options: { dryRun?: boolean } = {}
+): Promise<UpsertVariantsResult> {
+  if (products.length === 0) {
+    return { upserted: 0, new: 0, updated: 0, snapshotsCreated: 0 };
+  }
+
+  const { dryRun = false } = options;
+  let newCount = 0;
+  let updatedCount = 0;
+  let snapshotsCreated = 0;
+
+  for (const product of products) {
+    // Get the store_product_id for this product
+    const productResult = await pool.query(
+      `SELECT id FROM store_products
+       WHERE dispensary_id = $1 AND provider = $2 AND provider_product_id = $3`,
+      [dispensaryId, product.platform, product.externalProductId]
+    );
+
+    if (productResult.rows.length === 0) {
+      continue; // Product not found, skip variants
+    }
+
+    const storeProductId = productResult.rows[0].id;
+    const variants = extractVariantsFromRaw(product.rawProduct);
+
+    if (variants.length === 0) {
+      continue; // No variants to process
+    }
+
+    if (dryRun) {
+      console.log(`[DryRun] Would upsert ${variants.length} variants for product ${product.externalProductId}`);
+      continue;
+    }
+
+    for (const variant of variants) {
+      const { value: weightValue, unit: weightUnit } = parseWeight(variant.option);
+      const isOnSpecial = (variant.priceRecSpecial !== null && variant.priceRecSpecial < variant.priceRec) ||
+                          (variant.priceMedSpecial !== null && variant.priceMedSpecial < variant.priceMed);
+
+      // Upsert variant
+      const variantResult = await pool.query(
+        `INSERT INTO product_variants (
+          store_product_id, dispensary_id,
+          option, canonical_sku, canonical_id, canonical_name,
+          price_rec, price_med, price_rec_special, price_med_special,
+          quantity, quantity_available, in_stock, is_on_special,
+          weight_value, weight_unit,
+          first_seen_at, last_seen_at, updated_at
+        ) VALUES (
+          $1, $2,
+          $3, $4, $5, $6,
+          $7, $8, $9, $10,
+          $11, $11, $12, $13,
+          $14, $15,
+          NOW(), NOW(), NOW()
+        )
+        ON CONFLICT (store_product_id, option)
+        DO UPDATE SET
+          canonical_sku = COALESCE(EXCLUDED.canonical_sku, product_variants.canonical_sku),
+          canonical_id = COALESCE(EXCLUDED.canonical_id, product_variants.canonical_id),
+          canonical_name = COALESCE(EXCLUDED.canonical_name, product_variants.canonical_name),
+          price_rec = EXCLUDED.price_rec,
+          price_med = EXCLUDED.price_med,
+          price_rec_special = EXCLUDED.price_rec_special,
+          price_med_special = EXCLUDED.price_med_special,
+          quantity = EXCLUDED.quantity,
+          quantity_available = EXCLUDED.quantity_available,
+          in_stock = EXCLUDED.in_stock,
+          is_on_special = EXCLUDED.is_on_special,
+          weight_value = COALESCE(EXCLUDED.weight_value, product_variants.weight_value),
+          weight_unit = COALESCE(EXCLUDED.weight_unit, product_variants.weight_unit),
+          last_seen_at = NOW(),
+          last_price_change_at = CASE
+            WHEN product_variants.price_rec IS DISTINCT FROM EXCLUDED.price_rec
+              OR product_variants.price_rec_special IS DISTINCT FROM EXCLUDED.price_rec_special
+            THEN NOW()
+            ELSE product_variants.last_price_change_at
+          END,
+          last_stock_change_at = CASE
+            WHEN product_variants.quantity IS DISTINCT FROM EXCLUDED.quantity
+            THEN NOW()
+            ELSE product_variants.last_stock_change_at
+          END,
+          updated_at = NOW()
+        RETURNING id, (xmax = 0) as is_new`,
+        [
+          storeProductId, dispensaryId,
+          variant.option, variant.canonicalSku, variant.canonicalId, variant.canonicalName,
+          variant.priceRec, variant.priceMed, variant.priceRecSpecial, variant.priceMedSpecial,
+          variant.quantity, variant.inStock, isOnSpecial,
+          weightValue, weightUnit,
+        ]
+      );
+
+      const variantId = variantResult.rows[0].id;
+      if (variantResult.rows[0]?.is_new) {
+        newCount++;
+      } else {
+        updatedCount++;
+      }
+
+      // Create variant snapshot
+      await pool.query(
+        `INSERT INTO product_variant_snapshots (
+          product_variant_id, store_product_id, dispensary_id, crawl_run_id,
+          option,
+          price_rec, price_med, price_rec_special, price_med_special,
+          quantity, in_stock, is_on_special,
+          captured_at
+        ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, NOW())`,
+        [
+          variantId, storeProductId, dispensaryId, crawlRunId,
+          variant.option,
+          variant.priceRec, variant.priceMed, variant.priceRecSpecial, variant.priceMedSpecial,
+          variant.quantity, variant.inStock, isOnSpecial,
+        ]
+      );
+      snapshotsCreated++;
+    }
+  }
+
+  return {
+    upserted: newCount + updatedCount,
+    new: newCount,
+    updated: updatedCount,
+    snapshotsCreated,
+  };
+}
+
 // ============================================================
 // DISCONTINUED PRODUCTS
 // ============================================================
@@ -373,6 +571,9 @@ export interface HydratePayloadResult {
  productsDiscontinued: number;
  snapshotsCreated: number;
  brandsCreated: number;
+  variantsUpserted: number;
+  variantsNew: number;
+  variantSnapshotsCreated: number;
 }

 /**
@@ -399,7 +600,7 @@ export async function hydrateToCanonical(
    { dryRun }
  );

-  // 3. Create snapshots
+  // 3. Create product snapshots
  const snapshotResult = await createStoreProductSnapshots(
    pool,
    dispensaryId,
@@ -410,7 +611,16 @@ export async function hydrateToCanonical(
    { dryRun }
  );

-  // 4. Mark discontinued products
+  // 4. Upsert variants and create variant snapshots
+  const variantResult = await upsertProductVariants(
+    pool,
+    dispensaryId,
+    normResult.products,
+    crawlRunId,
+    { dryRun }
+  );
+
+  // 5. Mark discontinued products
  const currentProductIds = new Set(
    normResult.products.map((p) => p.externalProductId)
  );
@@ -431,5 +641,8 @@ export async function hydrateToCanonical(
    productsDiscontinued: discontinuedCount,
    snapshotsCreated: snapshotResult.created,
    brandsCreated: brandResult.new,
+    variantsUpserted: variantResult.upserted,
+    variantsNew: variantResult.new,
+    variantSnapshotsCreated: variantResult.snapshotsCreated,
  };
 }