Creationshop/cannaiq
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: SEO template library, discovery pipeline, and orchestrator enhancements

Browse Source 
## SEO Template Library
- Add complete template library with 7 page types (state, city, category, brand, product, search, regeneration)
- Add Template Library tab in SEO Orchestrator with accordion-based editors
- Add template preview, validation, and variable injection engine
- Add API endpoints: /api/seo/templates, preview, validate, generate, regenerate

## Discovery Pipeline
- Add promotion.ts for discovery location validation and promotion
- Add discover-all-states.ts script for multi-state discovery
- Add promotion log migration (067)
- Enhance discovery routes and types

## Orchestrator & Admin
- Add crawl_enabled filter to stores page
- Add API permissions page
- Add job queue management
- Add price analytics routes
- Add markets and intelligence routes
- Enhance dashboard and worker monitoring

## Infrastructure
- Add migrations for worker definitions, SEO settings, field alignment
- Add canonical pipeline for scraper v2
- Update hydration and sync orchestrator
- Enhance multi-state query service

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
Kelly
2025-12-09 00:05:34 -07:00
parent 9711d594db
commit 2f483b3084
83 changed files with 16700 additions and 1277 deletions
Download Patch File Download Diff File Expand all files Collapse all files

237
backend/src/hydration/canonical-upsert.ts
View File

@@ -68,7 +68,7 @@ export async function upsertStoreProducts(
const result = await client.query(
`INSERT INTO store_products (
dispensary_id, provider, provider_product_id, provider_brand_id,
name, brand_name, category, subcategory,
name_raw, brand_name_raw, category_raw, subcategory_raw,
price_rec, price_med, price_rec_special, price_med_special,
is_on_special, discount_percent,
is_in_stock, stock_status,
@@ -87,10 +87,10 @@ export async function upsertStoreProducts(
)
ON CONFLICT (dispensary_id, provider, provider_product_id)
DO UPDATE SET
name = EXCLUDED.name,
brand_name = EXCLUDED.brand_name,
category = EXCLUDED.category,
subcategory = EXCLUDED.subcategory,
name_raw = EXCLUDED.name_raw,
brand_name_raw = EXCLUDED.brand_name_raw,
category_raw = EXCLUDED.category_raw,
subcategory_raw = EXCLUDED.subcategory_raw,
price_rec = EXCLUDED.price_rec,
price_med = EXCLUDED.price_med,
price_rec_special = EXCLUDED.price_rec_special,
@@ -122,8 +122,9 @@ export async function upsertStoreProducts(
productPricing?.discountPercent,
productAvailability?.inStock ?? true,
productAvailability?.stockStatus || 'unknown',
product.thcPercent,
product.cbdPercent,
// Clamp THC/CBD to valid percentage range (0-100) - some products report mg as %
product.thcPercent !== null && product.thcPercent <= 100 ? product.thcPercent : null,
product.cbdPercent !== null && product.cbdPercent <= 100 ? product.cbdPercent : null,
product.primaryImageUrl,
]
);
@@ -212,8 +213,9 @@ export async function createStoreProductSnapshots(
productAvailability?.inStock ?? true,
productAvailability?.quantity,
productAvailability?.stockStatus || 'unknown',
product.thcPercent,
product.cbdPercent,
// Clamp THC/CBD to valid percentage range (0-100) - some products report mg as %
product.thcPercent !== null && product.thcPercent <= 100 ? product.thcPercent : null,
product.cbdPercent !== null && product.cbdPercent <= 100 ? product.cbdPercent : null,
product.primaryImageUrl,
JSON.stringify(product.rawProduct),
]);
@@ -229,7 +231,7 @@ export async function createStoreProductSnapshots(
`INSERT INTO store_product_snapshots (
dispensary_id, provider, provider_product_id, crawl_run_id,
captured_at,
name, brand_name, category, subcategory,
name_raw, brand_name_raw, category_raw, subcategory_raw,
price_rec, price_med, price_rec_special, price_med_special,
is_on_special, discount_percent,
is_in_stock, stock_quantity, stock_status,
@@ -245,6 +247,202 @@ export async function createStoreProductSnapshots(
return { created };
}
// ============================================================
// VARIANT UPSERTS
// ============================================================
export interface UpsertVariantsResult {
upserted: number;
new: number;
updated: number;
snapshotsCreated: number;
}
/**
* Extract variant data from raw Dutchie product
*/
function extractVariantsFromRaw(rawProduct: any): any[] {
const children = rawProduct?.POSMetaData?.children || [];
return children.map((child: any) => ({
option: child.option || child.key || '',
canonicalSku: child.canonicalSKU || null,
canonicalId: child.canonicalID || null,
canonicalName: child.canonicalName || null,
priceRec: child.recPrice || child.price || null,
priceMed: child.medPrice || null,
priceRecSpecial: child.recSpecialPrice || null,
priceMedSpecial: child.medSpecialPrice || null,
quantity: child.quantityAvailable ?? child.quantity ?? null,
inStock: (child.quantityAvailable ?? child.quantity ?? 0) > 0,
}));
}
/**
* Parse weight value and unit from option string
* e.g., "1g" -> { value: 1, unit: "g" }
* "3.5g" -> { value: 3.5, unit: "g" }
* "1/8oz" -> { value: 0.125, unit: "oz" }
*/
function parseWeight(option: string): { value: number | null; unit: string | null } {
if (!option) return { value: null, unit: null };
// Handle fractions like "1/8oz"
const fractionMatch = option.match(/^(\d+)\/(\d+)\s*(g|oz|mg|ml)?$/i);
if (fractionMatch) {
const value = parseInt(fractionMatch[1]) / parseInt(fractionMatch[2]);
return { value, unit: fractionMatch[3]?.toLowerCase() || 'oz' };
}
// Handle decimals like "3.5g" or "100mg"
const decimalMatch = option.match(/^([\d.]+)\s*(g|oz|mg|ml|each)?$/i);
if (decimalMatch) {
return {
value: parseFloat(decimalMatch[1]),
unit: decimalMatch[2]?.toLowerCase() || null
};
}
return { value: null, unit: null };
}
/**
* Upsert variants for products and create variant snapshots
*/
export async function upsertProductVariants(
pool: Pool,
dispensaryId: number,
products: NormalizedProduct[],
crawlRunId: number | null,
options: { dryRun?: boolean } = {}
): Promise<UpsertVariantsResult> {
if (products.length === 0) {
return { upserted: 0, new: 0, updated: 0, snapshotsCreated: 0 };
}
const { dryRun = false } = options;
let newCount = 0;
let updatedCount = 0;
let snapshotsCreated = 0;
for (const product of products) {
// Get the store_product_id for this product
const productResult = await pool.query(
`SELECT id FROM store_products
WHERE dispensary_id = $1 AND provider = $2 AND provider_product_id = $3`,
[dispensaryId, product.platform, product.externalProductId]
);
if (productResult.rows.length === 0) {
continue; // Product not found, skip variants
}
const storeProductId = productResult.rows[0].id;
const variants = extractVariantsFromRaw(product.rawProduct);
if (variants.length === 0) {
continue; // No variants to process
}
if (dryRun) {
console.log(`[DryRun] Would upsert ${variants.length} variants for product ${product.externalProductId}`);
continue;
}
for (const variant of variants) {
const { value: weightValue, unit: weightUnit } = parseWeight(variant.option);
const isOnSpecial = (variant.priceRecSpecial !== null && variant.priceRecSpecial < variant.priceRec) ||
(variant.priceMedSpecial !== null && variant.priceMedSpecial < variant.priceMed);
// Upsert variant
const variantResult = await pool.query(
`INSERT INTO product_variants (
store_product_id, dispensary_id,
option, canonical_sku, canonical_id, canonical_name,
price_rec, price_med, price_rec_special, price_med_special,
quantity, quantity_available, in_stock, is_on_special,
weight_value, weight_unit,
first_seen_at, last_seen_at, updated_at
) VALUES (
$1, $2,
$3, $4, $5, $6,
$7, $8, $9, $10,
$11, $11, $12, $13,
$14, $15,
NOW(), NOW(), NOW()
)
ON CONFLICT (store_product_id, option)
DO UPDATE SET
canonical_sku = COALESCE(EXCLUDED.canonical_sku, product_variants.canonical_sku),
canonical_id = COALESCE(EXCLUDED.canonical_id, product_variants.canonical_id),
canonical_name = COALESCE(EXCLUDED.canonical_name, product_variants.canonical_name),
price_rec = EXCLUDED.price_rec,
price_med = EXCLUDED.price_med,
price_rec_special = EXCLUDED.price_rec_special,
price_med_special = EXCLUDED.price_med_special,
quantity = EXCLUDED.quantity,
quantity_available = EXCLUDED.quantity_available,
in_stock = EXCLUDED.in_stock,
is_on_special = EXCLUDED.is_on_special,
weight_value = COALESCE(EXCLUDED.weight_value, product_variants.weight_value),
weight_unit = COALESCE(EXCLUDED.weight_unit, product_variants.weight_unit),
last_seen_at = NOW(),
last_price_change_at = CASE
WHEN product_variants.price_rec IS DISTINCT FROM EXCLUDED.price_rec
OR product_variants.price_rec_special IS DISTINCT FROM EXCLUDED.price_rec_special
THEN NOW()
ELSE product_variants.last_price_change_at
END,
last_stock_change_at = CASE
WHEN product_variants.quantity IS DISTINCT FROM EXCLUDED.quantity
THEN NOW()
ELSE product_variants.last_stock_change_at
END,
updated_at = NOW()
RETURNING id, (xmax = 0) as is_new`,
[
storeProductId, dispensaryId,
variant.option, variant.canonicalSku, variant.canonicalId, variant.canonicalName,
variant.priceRec, variant.priceMed, variant.priceRecSpecial, variant.priceMedSpecial,
variant.quantity, variant.inStock, isOnSpecial,
weightValue, weightUnit,
]
);
const variantId = variantResult.rows[0].id;
if (variantResult.rows[0]?.is_new) {
newCount++;
} else {
updatedCount++;
}
// Create variant snapshot
await pool.query(
`INSERT INTO product_variant_snapshots (
product_variant_id, store_product_id, dispensary_id, crawl_run_id,
option,
price_rec, price_med, price_rec_special, price_med_special,
quantity, in_stock, is_on_special,
captured_at
) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, NOW())`,
[
variantId, storeProductId, dispensaryId, crawlRunId,
variant.option,
variant.priceRec, variant.priceMed, variant.priceRecSpecial, variant.priceMedSpecial,
variant.quantity, variant.inStock, isOnSpecial,
]
);
snapshotsCreated++;
}
}
return {
upserted: newCount + updatedCount,
new: newCount,
updated: updatedCount,
snapshotsCreated,
};
}
// ============================================================
// DISCONTINUED PRODUCTS
// ============================================================
@@ -373,6 +571,9 @@ export interface HydratePayloadResult {
productsDiscontinued: number;
snapshotsCreated: number;
brandsCreated: number;
variantsUpserted: number;
variantsNew: number;
variantSnapshotsCreated: number;
}
/**
@@ -399,7 +600,7 @@ export async function hydrateToCanonical(
{ dryRun }
);
// 3. Create snapshots
// 3. Create product snapshots
const snapshotResult = await createStoreProductSnapshots(
pool,
dispensaryId,
@@ -410,7 +611,16 @@ export async function hydrateToCanonical(
{ dryRun }
);
// 4. Mark discontinued products
// 4. Upsert variants and create variant snapshots
const variantResult = await upsertProductVariants(
pool,
dispensaryId,
normResult.products,
crawlRunId,
{ dryRun }
);
// 5. Mark discontinued products
const currentProductIds = new Set(
normResult.products.map((p) => p.externalProductId)
);
@@ -431,5 +641,8 @@ export async function hydrateToCanonical(
productsDiscontinued: discontinuedCount,
snapshotsCreated: snapshotResult.created,
brandsCreated: brandResult.new,
variantsUpserted: variantResult.upserted,
variantsNew: variantResult.new,
variantSnapshotsCreated: variantResult.snapshotsCreated,
};
}

151
backend/src/hydration/incremental-sync.ts
View File

@@ -234,99 +234,94 @@ export async function syncProductsToCanonical(
const result = await pool.query(
`INSERT INTO store_products (
dispensary_id, state_id, provider, provider_product_id,
provider_brand_id, provider_dispensary_id, enterprise_product_id,
legacy_dutchie_product_id,
name, brand_name, category, subcategory, product_type, strain_type,
description, effects, cannabinoids,
thc_percent, cbd_percent, thc_content_text, cbd_content_text,
is_in_stock, stock_status, stock_quantity,
total_quantity_available, total_kiosk_quantity_available,
image_url, local_image_url, local_image_thumb_url, local_image_medium_url,
original_image_url, additional_images,
is_on_special, is_featured, medical_only, rec_only,
dispensary_id, provider, provider_product_id, provider_brand_id,
platform_dispensary_id, external_product_id,
name_raw, brand_name_raw, category_raw, subcategory_raw, strain_type,
description, effects, cannabinoids_v2,
thc_percent, cbd_percent, thc_content, cbd_content,
is_in_stock, stock_status, stock_quantity, total_quantity_available,
image_url, primary_image_url, images,
is_on_special, featured, medical_only, rec_only,
is_below_threshold, is_below_kiosk_threshold,
platform_status, c_name, weight, options, measurements,
first_seen_at, last_seen_at, updated_at
status, c_name, weight, measurements,
first_seen_at, last_seen_at, created_at, updated_at
) VALUES (
$1, $2, 'dutchie', $3,
$4, $5, $6,
$7,
$8, $9, $10, $11, $12, $13,
$14, $15, $16,
$17, $18, $19, $20,
$21, $22, $23,
$24, $25,
$26, $27, $28, $29,
$30, $31,
$32, $33, $34, $35,
$36, $37,
$38, $39, $40, $41, $42,
$43, $44, NOW()
$1, 'dutchie', $2, $3,
$4, $5,
$6, $7, $8, $9, $10,
$11, $12, $13,
$14, $15, $16, $17,
$18, $19, $20, $21,
$22, $23, $24,
$25, $26, $27, $28,
$29, $30,
$31, $32, $33, $34,
$35, $36, NOW(), NOW()
)
ON CONFLICT (dispensary_id, provider, provider_product_id)
DO UPDATE SET
legacy_dutchie_product_id = EXCLUDED.legacy_dutchie_product_id,
name = EXCLUDED.name,
brand_name = EXCLUDED.brand_name,
category = EXCLUDED.category,
subcategory = EXCLUDED.subcategory,
name_raw = EXCLUDED.name_raw,
brand_name_raw = EXCLUDED.brand_name_raw,
category_raw = EXCLUDED.category_raw,
subcategory_raw = EXCLUDED.subcategory_raw,
strain_type = EXCLUDED.strain_type,
is_in_stock = EXCLUDED.is_in_stock,
stock_status = EXCLUDED.stock_status,
stock_quantity = EXCLUDED.stock_quantity,
total_quantity_available = EXCLUDED.total_quantity_available,
thc_percent = EXCLUDED.thc_percent,
cbd_percent = EXCLUDED.cbd_percent,
thc_content = EXCLUDED.thc_content,
cbd_content = EXCLUDED.cbd_content,
image_url = EXCLUDED.image_url,
local_image_url = EXCLUDED.local_image_url,
primary_image_url = EXCLUDED.primary_image_url,
is_on_special = EXCLUDED.is_on_special,
platform_status = EXCLUDED.platform_status,
status = EXCLUDED.status,
description = COALESCE(EXCLUDED.description, store_products.description),
effects = COALESCE(EXCLUDED.effects, store_products.effects),
cannabinoids_v2 = COALESCE(EXCLUDED.cannabinoids_v2, store_products.cannabinoids_v2),
weight = EXCLUDED.weight,
measurements = EXCLUDED.measurements,
last_seen_at = NOW(),
updated_at = NOW()
RETURNING (xmax = 0) as is_new`,
[
dispensaryId,
stateId,
p.external_product_id,
p.brand_id,
p.platform_dispensary_id,
p.enterprise_product_id,
p.id,
p.name,
p.brand_name,
p.category || p.type,
p.subcategory,
p.type,
p.strain_type,
p.description,
p.effects,
p.cannabinoids_v2,
thcPercent,
cbdPercent,
p.thc_content,
p.cbd_content,
isInStock,
stockStatus,
p.total_quantity_available,
p.total_quantity_available,
p.total_kiosk_quantity_available,
p.primary_image_url,
p.local_image_url,
p.local_image_thumb_url,
p.local_image_medium_url,
p.original_image_url,
p.additional_images,
p.special || false,
p.featured || false,
p.medical_only || false,
p.rec_only || false,
p.is_below_threshold || false,
p.is_below_kiosk_threshold || false,
p.status,
p.c_name,
p.weight,
p.options,
p.measurements,
p.first_seen_at || p.updated_at,
p.last_seen_at || p.updated_at,
dispensaryId, // $1
p.external_product_id, // $2
p.brand_id, // $3
p.platform_dispensary_id, // $4
p.external_product_id, // $5 external_product_id
p.name, // $6
p.brand_name, // $7
p.type || p.category, // $8 category_raw
p.subcategory, // $9
p.strain_type, // $10
p.description, // $11
p.effects, // $12
p.cannabinoids_v2, // $13
thcPercent, // $14
cbdPercent, // $15
p.thc_content, // $16
p.cbd_content, // $17
isInStock, // $18
stockStatus, // $19
p.total_quantity_available || 0, // $20 stock_quantity
p.total_quantity_available || 0, // $21
p.primary_image_url, // $22 image_url
p.primary_image_url, // $23
p.additional_images, // $24 images
p.special || false, // $25
p.featured || false, // $26
p.medical_only || false, // $27
p.rec_only || false, // $28
p.is_below_threshold || false, // $29
p.is_below_kiosk_threshold || false, // $30
p.status, // $31
p.c_name, // $32
p.weight, // $33
p.measurements, // $34
p.first_seen_at || p.updated_at, // $35
p.last_seen_at || p.updated_at, // $36
]
);

3
backend/src/hydration/worker.ts
View File

@@ -107,7 +107,8 @@ export class HydrationWorker {
console.log(
`[HydrationWorker] ${this.options.dryRun ? '[DryRun] ' : ''}Processed payload ${payload.id}: ` +
`${hydrateResult.productsNew} new, ${hydrateResult.productsUpdated} updated, ` +
`${hydrateResult.productsDiscontinued} discontinued, ${hydrateResult.snapshotsCreated} snapshots`
`${hydrateResult.productsDiscontinued} discontinued, ${hydrateResult.snapshotsCreated} snapshots, ` +
`${hydrateResult.variantsUpserted} variants (${hydrateResult.variantSnapshotsCreated} variant snapshots)`
);
return {