Implements per-store high-frequency crawl scheduling and inventory snapshot tracking for sales velocity estimation (Hoodie Analytics parity). Database migrations: - 117: Per-store crawl_interval_minutes and next_crawl_at columns - 118: inventory_snapshots table (30-day retention) - 119: product_visibility_events table for OOS/brand alerts (90-day) Backend changes: - inventory-snapshots.ts: Shared utility normalizing Dutchie/Jane/Treez - visibility-events.ts: Detects OOS, price changes, brand drops - task-scheduler.ts: checkHighFrequencyStores() runs every 60s - Handler updates: 2-line additions to save snapshots/events API endpoints: - GET /api/tasks/schedules/high-frequency - PUT /api/tasks/schedules/high-frequency/:id - DELETE /api/tasks/schedules/high-frequency/:id Frontend: - TasksDashboard: Per-Store Schedules section with stats Features: - Per-store intervals (15/30/60 min configurable) - Jitter (0-20%) to avoid detection patterns - Cross-platform support (Dutchie, Jane, Treez) - No crawler core changes - scheduling/post-crawl only 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
375 lines
9.2 KiB
TypeScript
375 lines
9.2 KiB
TypeScript
/**
|
|
* Visibility Events Service
|
|
*
|
|
* Shared utility for detecting notable inventory events:
|
|
* - OOS (out of stock) - product disappeared from menu
|
|
* - Back in stock - product returned to menu
|
|
* - Brand dropped - brand no longer at store
|
|
* - Brand added - new brand at store
|
|
* - Price change - significant price change (>5%)
|
|
*
|
|
* Part of Real-Time Inventory Tracking feature.
|
|
*/
|
|
|
|
import { Pool } from 'pg';
|
|
import { Platform } from './inventory-snapshots';
|
|
|
|
export type EventType = 'oos' | 'back_in_stock' | 'brand_dropped' | 'brand_added' | 'price_change';
|
|
|
|
interface VisibilityEvent {
|
|
dispensary_id: number;
|
|
product_id: string | null;
|
|
product_name: string | null;
|
|
brand_name: string | null;
|
|
event_type: EventType;
|
|
previous_quantity: number | null;
|
|
previous_price: number | null;
|
|
new_price: number | null;
|
|
price_change_pct: number | null;
|
|
platform: Platform;
|
|
}
|
|
|
|
interface ProductInfo {
|
|
id: string;
|
|
name: string | null;
|
|
brand: string | null;
|
|
price: number | null;
|
|
quantity: number | null;
|
|
}
|
|
|
|
/**
|
|
* Extract product info from raw product based on platform.
|
|
*/
|
|
function extractProductInfo(product: any, platform: Platform): ProductInfo | null {
|
|
let id: string | null = null;
|
|
let name: string | null = null;
|
|
let brand: string | null = null;
|
|
let price: number | null = null;
|
|
let quantity: number | null = null;
|
|
|
|
switch (platform) {
|
|
case 'dutchie':
|
|
id = product.id;
|
|
name = product.Name || product.name;
|
|
brand = product.brand?.name || null;
|
|
price = product.recPrices?.[0] || product.Prices?.[0] || null;
|
|
if (product.children && Array.isArray(product.children)) {
|
|
quantity = product.children.reduce(
|
|
(sum: number, child: any) => sum + (child.quantityAvailable || child.quantity || 0),
|
|
0
|
|
);
|
|
}
|
|
break;
|
|
|
|
case 'jane':
|
|
id = String(product.product_id);
|
|
name = product.name;
|
|
brand = product.brand || null;
|
|
price = product.bucket_price || product.price_gram || null;
|
|
quantity = product.max_cart_quantity ?? null;
|
|
break;
|
|
|
|
case 'treez':
|
|
id = product.id;
|
|
name = product.name || product.menuTitle;
|
|
brand = product.brand || null;
|
|
price = product.customMinPrice ?? null;
|
|
quantity = product.availableUnits ?? null;
|
|
break;
|
|
}
|
|
|
|
if (!id) return null;
|
|
|
|
return { id, name, brand, price, quantity };
|
|
}
|
|
|
|
/**
|
|
* Detect visibility events by comparing current products to previous state.
|
|
*
|
|
* Call this after fetching products in any platform handler.
|
|
* Compares current products to stored state and creates events for changes.
|
|
*
|
|
* @param pool - Database connection pool
|
|
* @param dispensaryId - The dispensary ID
|
|
* @param products - Array of raw products from the platform
|
|
* @param platform - The platform type
|
|
* @returns Number of events created
|
|
*/
|
|
export async function detectVisibilityEvents(
|
|
pool: Pool,
|
|
dispensaryId: number,
|
|
products: any[],
|
|
platform: Platform
|
|
): Promise<number> {
|
|
if (!products || products.length === 0) {
|
|
return 0;
|
|
}
|
|
|
|
// Get previous product state from store_products
|
|
const { rows: previousProducts } = await pool.query(
|
|
`
|
|
SELECT
|
|
provider_product_id as id,
|
|
name,
|
|
brand,
|
|
price_rec as price
|
|
FROM store_products
|
|
WHERE dispensary_id = $1
|
|
`,
|
|
[dispensaryId]
|
|
);
|
|
|
|
// Build maps for comparison
|
|
const previousMap = new Map<string, { name: string; brand: string | null; price: number | null }>();
|
|
const previousBrands = new Set<string>();
|
|
|
|
for (const p of previousProducts) {
|
|
previousMap.set(p.id, { name: p.name, brand: p.brand, price: p.price });
|
|
if (p.brand) previousBrands.add(p.brand);
|
|
}
|
|
|
|
const currentMap = new Map<string, ProductInfo>();
|
|
const currentBrands = new Set<string>();
|
|
|
|
for (const product of products) {
|
|
const info = extractProductInfo(product, platform);
|
|
if (info) {
|
|
currentMap.set(info.id, info);
|
|
if (info.brand) currentBrands.add(info.brand);
|
|
}
|
|
}
|
|
|
|
const events: VisibilityEvent[] = [];
|
|
|
|
// Detect OOS events (products that disappeared)
|
|
Array.from(previousMap.entries()).forEach(([id, prev]) => {
|
|
if (!currentMap.has(id)) {
|
|
events.push({
|
|
dispensary_id: dispensaryId,
|
|
product_id: id,
|
|
product_name: prev.name,
|
|
brand_name: prev.brand,
|
|
event_type: 'oos',
|
|
previous_quantity: null,
|
|
previous_price: prev.price,
|
|
new_price: null,
|
|
price_change_pct: null,
|
|
platform,
|
|
});
|
|
}
|
|
});
|
|
|
|
// Detect back_in_stock events (products that appeared)
|
|
Array.from(currentMap.entries()).forEach(([id, curr]) => {
|
|
if (!previousMap.has(id)) {
|
|
events.push({
|
|
dispensary_id: dispensaryId,
|
|
product_id: id,
|
|
product_name: curr.name,
|
|
brand_name: curr.brand,
|
|
event_type: 'back_in_stock',
|
|
previous_quantity: null,
|
|
previous_price: null,
|
|
new_price: curr.price,
|
|
price_change_pct: null,
|
|
platform,
|
|
});
|
|
}
|
|
});
|
|
|
|
// Detect price changes (>5% change)
|
|
Array.from(currentMap.entries()).forEach(([id, curr]) => {
|
|
const prev = previousMap.get(id);
|
|
if (prev && prev.price != null && curr.price != null) {
|
|
const priceDiff = curr.price - prev.price;
|
|
const pctChange = (priceDiff / prev.price) * 100;
|
|
|
|
if (Math.abs(pctChange) >= 5) {
|
|
events.push({
|
|
dispensary_id: dispensaryId,
|
|
product_id: id,
|
|
product_name: curr.name,
|
|
brand_name: curr.brand,
|
|
event_type: 'price_change',
|
|
previous_quantity: null,
|
|
previous_price: prev.price,
|
|
new_price: curr.price,
|
|
price_change_pct: Math.round(pctChange * 100) / 100,
|
|
platform,
|
|
});
|
|
}
|
|
}
|
|
});
|
|
|
|
// Detect brand drops (brands that disappeared from store)
|
|
Array.from(previousBrands).forEach((brand) => {
|
|
if (!currentBrands.has(brand)) {
|
|
events.push({
|
|
dispensary_id: dispensaryId,
|
|
product_id: null,
|
|
product_name: null,
|
|
brand_name: brand,
|
|
event_type: 'brand_dropped',
|
|
previous_quantity: null,
|
|
previous_price: null,
|
|
new_price: null,
|
|
price_change_pct: null,
|
|
platform,
|
|
});
|
|
}
|
|
});
|
|
|
|
// Detect brand additions (new brands at store)
|
|
Array.from(currentBrands).forEach((brand) => {
|
|
if (!previousBrands.has(brand)) {
|
|
events.push({
|
|
dispensary_id: dispensaryId,
|
|
product_id: null,
|
|
product_name: null,
|
|
brand_name: brand,
|
|
event_type: 'brand_added',
|
|
previous_quantity: null,
|
|
previous_price: null,
|
|
new_price: null,
|
|
price_change_pct: null,
|
|
platform,
|
|
});
|
|
}
|
|
});
|
|
|
|
if (events.length === 0) {
|
|
return 0;
|
|
}
|
|
|
|
// Bulk insert events
|
|
const values: any[] = [];
|
|
const placeholders: string[] = [];
|
|
|
|
let paramIndex = 1;
|
|
for (const e of events) {
|
|
placeholders.push(
|
|
`($${paramIndex++}, $${paramIndex++}, $${paramIndex++}, $${paramIndex++}, $${paramIndex++}, $${paramIndex++}, $${paramIndex++}, $${paramIndex++}, $${paramIndex++}, $${paramIndex++})`
|
|
);
|
|
values.push(
|
|
e.dispensary_id,
|
|
e.product_id,
|
|
e.product_name,
|
|
e.brand_name,
|
|
e.event_type,
|
|
e.previous_quantity,
|
|
e.previous_price,
|
|
e.new_price,
|
|
e.price_change_pct,
|
|
e.platform
|
|
);
|
|
}
|
|
|
|
const query = `
|
|
INSERT INTO product_visibility_events (
|
|
dispensary_id,
|
|
product_id,
|
|
product_name,
|
|
brand_name,
|
|
event_type,
|
|
previous_quantity,
|
|
previous_price,
|
|
new_price,
|
|
price_change_pct,
|
|
platform
|
|
) VALUES ${placeholders.join(', ')}
|
|
`;
|
|
|
|
await pool.query(query, values);
|
|
|
|
console.log(`[VisibilityEvents] Created ${events.length} events for dispensary ${dispensaryId}`);
|
|
|
|
return events.length;
|
|
}
|
|
|
|
/**
|
|
* Get recent visibility events for a dispensary.
|
|
*/
|
|
export async function getRecentEvents(
|
|
pool: Pool,
|
|
dispensaryId: number,
|
|
limit = 100
|
|
): Promise<any[]> {
|
|
const { rows } = await pool.query(
|
|
`
|
|
SELECT
|
|
id,
|
|
product_id,
|
|
product_name,
|
|
brand_name,
|
|
event_type,
|
|
detected_at,
|
|
previous_price,
|
|
new_price,
|
|
price_change_pct,
|
|
platform,
|
|
notified,
|
|
acknowledged_at
|
|
FROM product_visibility_events
|
|
WHERE dispensary_id = $1
|
|
ORDER BY detected_at DESC
|
|
LIMIT $2
|
|
`,
|
|
[dispensaryId, limit]
|
|
);
|
|
|
|
return rows;
|
|
}
|
|
|
|
/**
|
|
* Get unnotified events for external system integration.
|
|
*/
|
|
export async function getUnnotifiedEvents(
|
|
pool: Pool,
|
|
limit = 100
|
|
): Promise<any[]> {
|
|
const { rows } = await pool.query(
|
|
`
|
|
SELECT
|
|
e.id,
|
|
e.dispensary_id,
|
|
d.name as dispensary_name,
|
|
e.product_id,
|
|
e.product_name,
|
|
e.brand_name,
|
|
e.event_type,
|
|
e.detected_at,
|
|
e.previous_price,
|
|
e.new_price,
|
|
e.price_change_pct,
|
|
e.platform
|
|
FROM product_visibility_events e
|
|
JOIN dispensaries d ON d.id = e.dispensary_id
|
|
WHERE e.notified = FALSE
|
|
ORDER BY e.detected_at DESC
|
|
LIMIT $1
|
|
`,
|
|
[limit]
|
|
);
|
|
|
|
return rows;
|
|
}
|
|
|
|
/**
|
|
* Mark events as notified.
|
|
*/
|
|
export async function markEventsNotified(
|
|
pool: Pool,
|
|
eventIds: number[]
|
|
): Promise<void> {
|
|
if (eventIds.length === 0) return;
|
|
|
|
await pool.query(
|
|
`
|
|
UPDATE product_visibility_events
|
|
SET notified = TRUE
|
|
WHERE id = ANY($1)
|
|
`,
|
|
[eventIds]
|
|
);
|
|
}
|