diff --git a/backend/migrations/120_daily_baseline_tracking.sql b/backend/migrations/120_daily_baseline_tracking.sql new file mode 100644 index 00000000..0e238fd9 --- /dev/null +++ b/backend/migrations/120_daily_baseline_tracking.sql @@ -0,0 +1,13 @@ +-- Migration 120: Daily baseline tracking +-- Track when each store's daily baseline payload was last saved +-- Part of Real-Time Inventory Tracking feature + +-- Add column to track last baseline save time +ALTER TABLE dispensaries ADD COLUMN IF NOT EXISTS last_baseline_at TIMESTAMPTZ DEFAULT NULL; + +-- Index for finding stores that need baselines +CREATE INDEX IF NOT EXISTS idx_dispensaries_baseline ON dispensaries(last_baseline_at) +WHERE crawl_enabled = TRUE; + +-- Comment +COMMENT ON COLUMN dispensaries.last_baseline_at IS 'Timestamp of last daily baseline payload save. Baselines saved once per day between 12:01 AM - 3:00 AM.'; diff --git a/backend/src/services/task-scheduler.ts b/backend/src/services/task-scheduler.ts index b9e463b0..8aff0e0c 100644 --- a/backend/src/services/task-scheduler.ts +++ b/backend/src/services/task-scheduler.ts @@ -688,6 +688,7 @@ class TaskScheduler { next_crawl_at: Date | null; last_crawl_started_at: Date | null; last_fetch_at: Date | null; + last_baseline_at: Date | null; inventory_changes_24h: number; price_changes_24h: number; }[]> { @@ -701,6 +702,7 @@ class TaskScheduler { next_crawl_at, last_crawl_started_at, last_fetch_at, + last_baseline_at, COALESCE(inventory_changes_24h, 0) as inventory_changes_24h, COALESCE(price_changes_24h, 0) as price_changes_24h FROM dispensaries diff --git a/backend/src/tasks/handlers/product-discovery-dutchie.ts b/backend/src/tasks/handlers/product-discovery-dutchie.ts index e345ad34..6deec3bc 100644 --- a/backend/src/tasks/handlers/product-discovery-dutchie.ts +++ b/backend/src/tasks/handlers/product-discovery-dutchie.ts @@ -22,7 +22,7 @@ */ import { TaskContext, TaskResult } from '../task-worker'; -import { saveRawPayload } from '../../utils/payload-storage'; +import { saveDailyBaseline } from '../../utils/payload-storage'; import { taskService } from '../task-service'; import { saveInventorySnapshots } from '../../services/inventory-snapshots'; import { detectVisibilityEvents } from '../../services/visibility-events'; @@ -367,7 +367,9 @@ export async function handleProductDiscoveryDutchie(ctx: TaskContext): Promise p.raw); @@ -130,28 +134,35 @@ export async function handleProductDiscoveryJane(ctx: TaskContext): Promise= BASELINE_WINDOW_START_MINUTE; + } else if (hours > BASELINE_WINDOW_START_HOUR && hours < BASELINE_WINDOW_END_HOUR) { + // Between 1am and 3am + return true; + } else if (hours === BASELINE_WINDOW_END_HOUR && minutes === BASELINE_WINDOW_END_MINUTE) { + // Exactly 3:00 AM - still included + return true; + } + + return false; +} + +/** + * Check if a store already has a baseline for today (same calendar day) + * + * @param pool - Database connection pool + * @param dispensaryId - ID of the dispensary + * @param now - Optional date to check against (defaults to current time) + * @returns true if baseline already exists for today + */ +export async function hasBaselineToday( + pool: Pool, + dispensaryId: number, + now: Date = new Date() +): Promise { + const result = await pool.query(` + SELECT last_baseline_at + FROM dispensaries + WHERE id = $1 + `, [dispensaryId]); + + if (result.rows.length === 0 || !result.rows[0].last_baseline_at) { + return false; + } + + const lastBaseline = new Date(result.rows[0].last_baseline_at); + + // Check if same calendar day + return lastBaseline.getFullYear() === now.getFullYear() && + lastBaseline.getMonth() === now.getMonth() && + lastBaseline.getDate() === now.getDate(); +} + +/** + * Result from daily baseline check + */ +export interface BaselineCheckResult { + shouldSave: boolean; + reason: 'saved' | 'outside_window' | 'already_exists'; + inWindow: boolean; + hasExisting: boolean; +} + +/** + * Check if a daily baseline should be saved for this store + * + * @param pool - Database connection pool + * @param dispensaryId - ID of the dispensary + * @returns BaselineCheckResult with save decision and reason + */ +export async function shouldSaveBaseline( + pool: Pool, + dispensaryId: number +): Promise { + const now = new Date(); + const inWindow = isInBaselineWindow(now); + const hasExisting = await hasBaselineToday(pool, dispensaryId, now); + + let shouldSave = false; + let reason: 'saved' | 'outside_window' | 'already_exists'; + + if (!inWindow) { + reason = 'outside_window'; + } else if (hasExisting) { + reason = 'already_exists'; + } else { + shouldSave = true; + reason = 'saved'; + } + + return { shouldSave, reason, inWindow, hasExisting }; +} + +/** + * Save a daily baseline payload (full payload) if conditions are met + * + * Conditions: + * 1. Current time is within baseline window (12:01 AM - 3:00 AM) + * 2. No baseline exists for this store today + * + * If conditions not met, returns null (payload not saved). + * Inventory snapshots should still be saved separately via saveInventorySnapshots(). + * + * @param pool - Database connection pool + * @param dispensaryId - ID of the dispensary + * @param payload - Raw JSON payload from GraphQL/API + * @param crawlRunId - Optional crawl_run ID for linking + * @param productCount - Number of products in payload + * @param platform - Platform identifier ('dutchie' | 'jane' | 'treez') + * @param taskId - Optional task ID for traceability in filename + * @returns SavePayloadResult if saved, null if skipped + */ +export async function saveDailyBaseline( + pool: Pool, + dispensaryId: number, + payload: any, + crawlRunId: number | null = null, + productCount: number = 0, + platform: string = 'dutchie', + taskId: number | null = null +): Promise { + const check = await shouldSaveBaseline(pool, dispensaryId); + + if (!check.shouldSave) { + console.log(`[PayloadStorage] Skipping baseline for store ${dispensaryId}: ${check.reason} (inWindow=${check.inWindow}, hasExisting=${check.hasExisting})`); + return null; + } + + // Save the full payload + const result = await saveRawPayload(pool, dispensaryId, payload, crawlRunId, productCount, platform, taskId); + + // Update last_baseline_at timestamp + await pool.query(` + UPDATE dispensaries + SET last_baseline_at = NOW() + WHERE id = $1 + `, [dispensaryId]); + + console.log(`[PayloadStorage] Saved daily baseline for store ${dispensaryId}: ${result.storagePath}`); + + return result; +} + +/** + * Get baseline status for a store (for dashboard display) + * + * @param pool - Database connection pool + * @param dispensaryId - ID of the dispensary + * @returns Baseline status info + */ +export async function getBaselineStatus( + pool: Pool, + dispensaryId: number +): Promise<{ + lastBaselineAt: Date | null; + hasBaselineToday: boolean; + inBaselineWindow: boolean; + nextWindowStart: Date; +}> { + const result = await pool.query(` + SELECT last_baseline_at + FROM dispensaries + WHERE id = $1 + `, [dispensaryId]); + + const lastBaselineAt = result.rows[0]?.last_baseline_at || null; + const now = new Date(); + const inWindow = isInBaselineWindow(now); + const hasToday = lastBaselineAt ? await hasBaselineToday(pool, dispensaryId, now) : false; + + // Calculate next window start + const nextWindowStart = new Date(now); + if (now.getHours() >= BASELINE_WINDOW_END_HOUR || + (now.getHours() === BASELINE_WINDOW_START_HOUR && now.getMinutes() < BASELINE_WINDOW_START_MINUTE)) { + // Before today's window or after today's window - next is tomorrow at 00:01 + if (now.getHours() >= BASELINE_WINDOW_END_HOUR) { + nextWindowStart.setDate(nextWindowStart.getDate() + 1); + } + nextWindowStart.setHours(BASELINE_WINDOW_START_HOUR, BASELINE_WINDOW_START_MINUTE, 0, 0); + } else { + // Currently in window - "next" is now + nextWindowStart.setHours(BASELINE_WINDOW_START_HOUR, BASELINE_WINDOW_START_MINUTE, 0, 0); + } + + return { + lastBaselineAt, + hasBaselineToday: hasToday, + inBaselineWindow: inWindow, + nextWindowStart + }; +} + /** * Delete old payloads (for retention policy) * diff --git a/cannaiq/src/lib/api.ts b/cannaiq/src/lib/api.ts index 84edc1b6..1e39383c 100755 --- a/cannaiq/src/lib/api.ts +++ b/cannaiq/src/lib/api.ts @@ -3280,6 +3280,7 @@ export interface HighFrequencyStore { next_crawl_at: string | null; last_crawl_started_at: string | null; last_fetch_at: string | null; + last_baseline_at: string | null; inventory_changes_24h: number; price_changes_24h: number; } diff --git a/cannaiq/src/pages/TasksDashboard.tsx b/cannaiq/src/pages/TasksDashboard.tsx index 01d62dd2..adc3e6a8 100644 --- a/cannaiq/src/pages/TasksDashboard.tsx +++ b/cannaiq/src/pages/TasksDashboard.tsx @@ -1806,7 +1806,7 @@ export default function TasksDashboard() {
{/* Stats Summary */} {highFreqStats.totalStores > 0 && ( -
+
By Interval:{' '} {Object.entries(highFreqStats.byInterval).map(([interval, count]) => ( @@ -1823,6 +1823,10 @@ export default function TasksDashboard() { ))}
+
+ + Daily baselines: 12:01 AM - 3:00 AM +
)} @@ -1852,6 +1856,9 @@ export default function TasksDashboard() { Last Fetch + + Baseline + Changes (24h) @@ -1889,6 +1896,24 @@ export default function TasksDashboard() { {store.last_fetch_at ? formatTimeAgo(store.last_fetch_at) : '-'} + + {store.last_baseline_at ? ( + + {new Date(store.last_baseline_at).toDateString() === new Date().toDateString() ? ( + + ) : ( + + )} + {formatTimeAgo(store.last_baseline_at)} + + ) : ( + Never + )} + {(store.inventory_changes_24h > 0 || store.price_changes_24h > 0) ? (