Creationshop/cannaiq
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: Add daily baseline payload logic (12:01 AM - 3:00 AM window)

Browse Source 
- Replace saveRawPayload with saveDailyBaseline in all handlers
- Full payloads only saved once per day per store during window
- Inventory snapshots still saved every crawl (lightweight tracking)
- Add last_baseline_at column to dispensaries table
- Show baseline status in Per-Store Schedules dashboard
- Display baseline window info (12:01 AM - 3:00 AM) in UI

Reduces storage ~95% for high-frequency stores while maintaining
full audit capability via daily baselines.

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
Kelly
2025-12-14 16:24:41 -07:00
parent 294d3db7a2
commit 35d6a17740
8 changed files with 335 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
backend/migrations/120_daily_baseline_tracking.sql Normal file
View File

@@ -0,0 +1,13 @@
-- Migration 120: Daily baseline tracking
-- Track when each store's daily baseline payload was last saved
-- Part of Real-Time Inventory Tracking feature
-- Add column to track last baseline save time
ALTER TABLE dispensaries ADD COLUMN IF NOT EXISTS last_baseline_at TIMESTAMPTZ DEFAULT NULL;
-- Index for finding stores that need baselines
CREATE INDEX IF NOT EXISTS idx_dispensaries_baseline ON dispensaries(last_baseline_at)
WHERE crawl_enabled = TRUE;
-- Comment
COMMENT ON COLUMN dispensaries.last_baseline_at IS 'Timestamp of last daily baseline payload save. Baselines saved once per day between 12:01 AM - 3:00 AM.';

2
backend/src/services/task-scheduler.ts
View File

@@ -688,6 +688,7 @@ class TaskScheduler {
next_crawl_at: Date | null;
last_crawl_started_at: Date | null;
last_fetch_at: Date | null;
last_baseline_at: Date | null;
inventory_changes_24h: number;
price_changes_24h: number;
}[]> {
@@ -701,6 +702,7 @@ class TaskScheduler {
next_crawl_at,
last_crawl_started_at,
last_fetch_at,
last_baseline_at,
COALESCE(inventory_changes_24h, 0) as inventory_changes_24h,
COALESCE(price_changes_24h, 0) as price_changes_24h
FROM dispensaries

44
backend/src/tasks/handlers/product-discovery-dutchie.ts
View File

@@ -22,7 +22,7 @@
*/
import { TaskContext, TaskResult } from '../task-worker';
import { saveRawPayload } from '../../utils/payload-storage';
import { saveDailyBaseline } from '../../utils/payload-storage';
import { taskService } from '../task-service';
import { saveInventorySnapshots } from '../../services/inventory-snapshots';
import { detectVisibilityEvents } from '../../services/visibility-events';
@@ -367,7 +367,9 @@ export async function handleProductDiscoveryDutchie(ctx: TaskContext): Promise<T
await ctx.heartbeat();
// ============================================================
// STEP 5: Save raw payload to filesystem
// STEP 5: Save daily baseline (full payload) if in window
// Daily baselines are saved once per day per store (12:01 AM - 3:00 AM)
// Outside this window, only inventory snapshots are saved (Step 5.5)
// ============================================================
updateStep('saving', `Saving ${result.products.length} products`);
const rawPayload = {
@@ -379,7 +381,8 @@ export async function handleProductDiscoveryDutchie(ctx: TaskContext): Promise<T
products: result.products,
};
const payloadResult = await saveRawPayload(
// saveDailyBaseline returns null if outside window or baseline already exists today
const payloadResult = await saveDailyBaseline(
pool,
dispensaryId,
rawPayload,
@@ -389,7 +392,11 @@ export async function handleProductDiscoveryDutchie(ctx: TaskContext): Promise<T
task.id // task ID for traceability
);
console.log(`[ProductDiscoveryHTTP] Saved payload #${payloadResult.id} (${(payloadResult.sizeBytes / 1024).toFixed(1)}KB)`);
if (payloadResult) {
console.log(`[ProductDiscoveryHTTP] Saved daily baseline #${payloadResult.id} (${(payloadResult.sizeBytes / 1024).toFixed(1)}KB)`);
} else {
console.log(`[ProductDiscoveryHTTP] Skipped full payload save (outside baseline window or already exists)`);
}
// ============================================================
// STEP 5.5: Save inventory snapshots and detect visibility events
@@ -412,16 +419,20 @@ export async function handleProductDiscoveryDutchie(ctx: TaskContext): Promise<T
// ============================================================
// STEP 7: Queue product_refresh task to process the payload
// Only queue if a baseline payload was saved (need payload_id)
// ============================================================
await taskService.createTask({
role: 'product_refresh',
dispensary_id: dispensaryId,
priority: task.priority || 0,
method: 'http', // Browser-only transport
payload: { payload_id: payloadResult.id },
});
console.log(`[ProductDiscoveryHTTP] Queued product_refresh task for payload #${payloadResult.id}`);
if (payloadResult) {
await taskService.createTask({
role: 'product_refresh',
dispensary_id: dispensaryId,
priority: task.priority || 0,
method: 'http', // Browser-only transport
payload: { payload_id: payloadResult.id },
});
console.log(`[ProductDiscoveryHTTP] Queued product_refresh task for payload #${payloadResult.id}`);
} else {
console.log(`[ProductDiscoveryHTTP] Skipped product_refresh (no payload saved)`);
}
// ============================================================
// STEP 8: Stage checkpoint - observational update
@@ -444,9 +455,12 @@ export async function handleProductDiscoveryDutchie(ctx: TaskContext): Promise<T
return {
success: true,
payloadId: payloadResult.id,
payloadId: payloadResult?.id || null,
productCount: result.products.length,
sizeBytes: payloadResult.sizeBytes,
sizeBytes: payloadResult?.sizeBytes || 0,
baselineSaved: !!payloadResult,
snapshotCount,
eventCount,
};
} catch (error: unknown) {

45
backend/src/tasks/handlers/product-discovery-jane.ts
View File

@@ -15,7 +15,7 @@ import {
setCrawlRotator,
fetchProductsByStoreIdDirect,
} from '../../platforms/jane';
import { saveRawPayload } from '../../utils/payload-storage';
import { saveDailyBaseline } from '../../utils/payload-storage';
import { taskService } from '../task-service';
import { saveInventorySnapshots } from '../../services/inventory-snapshots';
import { detectVisibilityEvents } from '../../services/visibility-events';
@@ -99,8 +99,8 @@ export async function handleProductDiscoveryJane(ctx: TaskContext): Promise<Task
storeId: dispensary.platform_dispensary_id,
};
// Save raw payload to filesystem (platform = 'jane')
const { id: payloadId, sizeBytes } = await saveRawPayload(
// Save daily baseline to filesystem (only in 12:01-3:00 AM window, once per day)
const payloadResult = await saveDailyBaseline(
pool,
dispensaryId,
rawPayload,
@@ -110,7 +110,11 @@ export async function handleProductDiscoveryJane(ctx: TaskContext): Promise<Task
task.id // task ID for traceability
);
console.log(`[JaneProductDiscovery] Saved payload ${payloadId} (${Math.round(sizeBytes / 1024)}KB)`);
if (payloadResult) {
console.log(`[JaneProductDiscovery] Saved daily baseline ${payloadResult.id} (${Math.round(payloadResult.sizeBytes / 1024)}KB)`);
} else {
console.log(`[JaneProductDiscovery] Skipped full payload save (outside baseline window or already exists)`);
}
// Save inventory snapshots and detect visibility events
const rawProducts = result.products.map(p => p.raw);
@@ -130,28 +134,35 @@ export async function handleProductDiscoveryJane(ctx: TaskContext): Promise<Task
[dispensaryId]
);
// Queue product_refresh task for normalization
console.log(`[JaneProductDiscovery] Queuing product_refresh for payload ${payloadId}`);
await taskService.createTask({
role: 'product_refresh',
dispensary_id: dispensaryId,
platform: 'jane',
// method undefined = any worker can process (product_refresh is local)
priority: task.priority || 0,
payload: { payload_id: payloadId },
});
// Queue product_refresh task for normalization (only if baseline payload was saved)
if (payloadResult) {
console.log(`[JaneProductDiscovery] Queuing product_refresh for payload ${payloadResult.id}`);
await taskService.createTask({
role: 'product_refresh',
dispensary_id: dispensaryId,
platform: 'jane',
// method undefined = any worker can process (product_refresh is local)
priority: task.priority || 0,
payload: { payload_id: payloadResult.id },
});
} else {
console.log(`[JaneProductDiscovery] Skipped product_refresh (no payload saved)`);
}
return {
success: true,
productCount: result.products.length,
payloadId,
payloadSizeKB: Math.round(sizeBytes / 1024),
payloadId: payloadResult?.id || null,
payloadSizeKB: payloadResult ? Math.round(payloadResult.sizeBytes / 1024) : 0,
baselineSaved: !!payloadResult,
snapshotCount,
eventCount,
storeInfo: result.store ? {
id: result.store.id,
name: result.store.name,
productCount: result.store.product_count,
} : null,
queuedProductRefresh: true,
queuedProductRefresh: !!payloadResult,
};
} catch (error: unknown) {
const errorMessage = error instanceof Error ? error.message : 'Unknown error';

43
backend/src/tasks/handlers/product-discovery-treez.ts
View File

@@ -29,7 +29,7 @@ import {
setCrawlRotator,
fetchProductsByStoreId,
} from '../../platforms/treez';
import { saveRawPayload } from '../../utils/payload-storage';
import { saveDailyBaseline } from '../../utils/payload-storage';
import { taskService } from '../task-service';
import { saveInventorySnapshots } from '../../services/inventory-snapshots';
import { detectVisibilityEvents } from '../../services/visibility-events';
@@ -116,8 +116,8 @@ export async function handleProductDiscoveryTreez(ctx: TaskContext): Promise<Tas
dispensaryId,
};
// Save raw payload to filesystem (platform = 'treez')
const { id: payloadId, sizeBytes } = await saveRawPayload(
// Save daily baseline to filesystem (only in 12:01-3:00 AM window, once per day)
const payloadResult = await saveDailyBaseline(
pool,
dispensaryId,
rawPayload,
@@ -127,7 +127,11 @@ export async function handleProductDiscoveryTreez(ctx: TaskContext): Promise<Tas
task.id // task ID for traceability
);
console.log(`[TreezProductDiscovery] Saved payload ${payloadId} (${Math.round(sizeBytes / 1024)}KB)`);
if (payloadResult) {
console.log(`[TreezProductDiscovery] Saved daily baseline ${payloadResult.id} (${Math.round(payloadResult.sizeBytes / 1024)}KB)`);
} else {
console.log(`[TreezProductDiscovery] Skipped full payload save (outside baseline window or already exists)`);
}
// Save inventory snapshots and detect visibility events
const snapshotCount = await saveInventorySnapshots(pool, dispensaryId, result.products, 'treez');
@@ -147,24 +151,31 @@ export async function handleProductDiscoveryTreez(ctx: TaskContext): Promise<Tas
[dispensaryId, result.products.length]
);
// Queue Treez-specific product_refresh task for normalization
console.log(`[TreezProductDiscovery] Queuing product_refresh_treez for payload ${payloadId}`);
await taskService.createTask({
role: 'product_refresh_treez',
dispensary_id: dispensaryId,
platform: 'treez',
priority: task.priority || 0,
payload: { payload_id: payloadId },
});
// Queue Treez-specific product_refresh task for normalization (only if baseline payload was saved)
if (payloadResult) {
console.log(`[TreezProductDiscovery] Queuing product_refresh_treez for payload ${payloadResult.id}`);
await taskService.createTask({
role: 'product_refresh_treez',
dispensary_id: dispensaryId,
platform: 'treez',
priority: task.priority || 0,
payload: { payload_id: payloadResult.id },
});
} else {
console.log(`[TreezProductDiscovery] Skipped product_refresh (no payload saved)`);
}
return {
success: true,
productCount: result.totalCaptured,
payloadId,
payloadSizeKB: Math.round(sizeBytes / 1024),
payloadId: payloadResult?.id || null,
payloadSizeKB: payloadResult ? Math.round(payloadResult.sizeBytes / 1024) : 0,
baselineSaved: !!payloadResult,
snapshotCount,
eventCount,
storeId: result.storeId,
sourceUrl: result.sourceUrl,
queuedProductRefresh: true,
queuedProductRefresh: !!payloadResult,
};
} catch (error: unknown) {
const errorMessage = error instanceof Error ? error.message : 'Unknown error';

209
backend/src/utils/payload-storage.ts
View File

@@ -616,6 +616,215 @@ export async function getLatestDiscoveryPayload(
};
}
/**
* Daily Baseline Configuration
*
* Baselines are full payload saves that happen once per day per store.
* Window: 12:01 AM - 3:00 AM (configurable)
*
* Outside this window, only inventory snapshots are saved (lightweight tracking).
* This reduces storage by ~95% while maintaining full audit capability via daily baselines.
*/
const BASELINE_WINDOW_START_HOUR = 0; // 12:00 AM
const BASELINE_WINDOW_START_MINUTE = 1; // 12:01 AM
const BASELINE_WINDOW_END_HOUR = 3; // 3:00 AM
const BASELINE_WINDOW_END_MINUTE = 0; // 3:00 AM
/**
* Check if current time is within the daily baseline window (12:01 AM - 3:00 AM)
*
* @param now - Optional date to check (defaults to current time)
* @returns true if within baseline window
*/
export function isInBaselineWindow(now: Date = new Date()): boolean {
const hours = now.getHours();
const minutes = now.getMinutes();
// Check if time is between 00:01 and 03:00
if (hours === BASELINE_WINDOW_START_HOUR) {
// Between midnight and 1am, must be after 00:01
return minutes >= BASELINE_WINDOW_START_MINUTE;
} else if (hours > BASELINE_WINDOW_START_HOUR && hours < BASELINE_WINDOW_END_HOUR) {
// Between 1am and 3am
return true;
} else if (hours === BASELINE_WINDOW_END_HOUR && minutes === BASELINE_WINDOW_END_MINUTE) {
// Exactly 3:00 AM - still included
return true;
}
return false;
}
/**
* Check if a store already has a baseline for today (same calendar day)
*
* @param pool - Database connection pool
* @param dispensaryId - ID of the dispensary
* @param now - Optional date to check against (defaults to current time)
* @returns true if baseline already exists for today
*/
export async function hasBaselineToday(
pool: Pool,
dispensaryId: number,
now: Date = new Date()
): Promise<boolean> {
const result = await pool.query(`
SELECT last_baseline_at
FROM dispensaries
WHERE id = $1
`, [dispensaryId]);
if (result.rows.length === 0 || !result.rows[0].last_baseline_at) {
return false;
}
const lastBaseline = new Date(result.rows[0].last_baseline_at);
// Check if same calendar day
return lastBaseline.getFullYear() === now.getFullYear() &&
lastBaseline.getMonth() === now.getMonth() &&
lastBaseline.getDate() === now.getDate();
}
/**
* Result from daily baseline check
*/
export interface BaselineCheckResult {
shouldSave: boolean;
reason: 'saved' | 'outside_window' | 'already_exists';
inWindow: boolean;
hasExisting: boolean;
}
/**
* Check if a daily baseline should be saved for this store
*
* @param pool - Database connection pool
* @param dispensaryId - ID of the dispensary
* @returns BaselineCheckResult with save decision and reason
*/
export async function shouldSaveBaseline(
pool: Pool,
dispensaryId: number
): Promise<BaselineCheckResult> {
const now = new Date();
const inWindow = isInBaselineWindow(now);
const hasExisting = await hasBaselineToday(pool, dispensaryId, now);
let shouldSave = false;
let reason: 'saved' | 'outside_window' | 'already_exists';
if (!inWindow) {
reason = 'outside_window';
} else if (hasExisting) {
reason = 'already_exists';
} else {
shouldSave = true;
reason = 'saved';
}
return { shouldSave, reason, inWindow, hasExisting };
}
/**
* Save a daily baseline payload (full payload) if conditions are met
*
* Conditions:
* 1. Current time is within baseline window (12:01 AM - 3:00 AM)
* 2. No baseline exists for this store today
*
* If conditions not met, returns null (payload not saved).
* Inventory snapshots should still be saved separately via saveInventorySnapshots().
*
* @param pool - Database connection pool
* @param dispensaryId - ID of the dispensary
* @param payload - Raw JSON payload from GraphQL/API
* @param crawlRunId - Optional crawl_run ID for linking
* @param productCount - Number of products in payload
* @param platform - Platform identifier ('dutchie' | 'jane' | 'treez')
* @param taskId - Optional task ID for traceability in filename
* @returns SavePayloadResult if saved, null if skipped
*/
export async function saveDailyBaseline(
pool: Pool,
dispensaryId: number,
payload: any,
crawlRunId: number | null = null,
productCount: number = 0,
platform: string = 'dutchie',
taskId: number | null = null
): Promise<SavePayloadResult | null> {
const check = await shouldSaveBaseline(pool, dispensaryId);
if (!check.shouldSave) {
console.log(`[PayloadStorage] Skipping baseline for store ${dispensaryId}: ${check.reason} (inWindow=${check.inWindow}, hasExisting=${check.hasExisting})`);
return null;
}
// Save the full payload
const result = await saveRawPayload(pool, dispensaryId, payload, crawlRunId, productCount, platform, taskId);
// Update last_baseline_at timestamp
await pool.query(`
UPDATE dispensaries
SET last_baseline_at = NOW()
WHERE id = $1
`, [dispensaryId]);
console.log(`[PayloadStorage] Saved daily baseline for store ${dispensaryId}: ${result.storagePath}`);
return result;
}
/**
* Get baseline status for a store (for dashboard display)
*
* @param pool - Database connection pool
* @param dispensaryId - ID of the dispensary
* @returns Baseline status info
*/
export async function getBaselineStatus(
pool: Pool,
dispensaryId: number
): Promise<{
lastBaselineAt: Date | null;
hasBaselineToday: boolean;
inBaselineWindow: boolean;
nextWindowStart: Date;
}> {
const result = await pool.query(`
SELECT last_baseline_at
FROM dispensaries
WHERE id = $1
`, [dispensaryId]);
const lastBaselineAt = result.rows[0]?.last_baseline_at || null;
const now = new Date();
const inWindow = isInBaselineWindow(now);
const hasToday = lastBaselineAt ? await hasBaselineToday(pool, dispensaryId, now) : false;
// Calculate next window start
const nextWindowStart = new Date(now);
if (now.getHours() >= BASELINE_WINDOW_END_HOUR ||
(now.getHours() === BASELINE_WINDOW_START_HOUR && now.getMinutes() < BASELINE_WINDOW_START_MINUTE)) {
// Before today's window or after today's window - next is tomorrow at 00:01
if (now.getHours() >= BASELINE_WINDOW_END_HOUR) {
nextWindowStart.setDate(nextWindowStart.getDate() + 1);
}
nextWindowStart.setHours(BASELINE_WINDOW_START_HOUR, BASELINE_WINDOW_START_MINUTE, 0, 0);
} else {
// Currently in window - "next" is now
nextWindowStart.setHours(BASELINE_WINDOW_START_HOUR, BASELINE_WINDOW_START_MINUTE, 0, 0);
}
return {
lastBaselineAt,
hasBaselineToday: hasToday,
inBaselineWindow: inWindow,
nextWindowStart
};
}
/**
* Delete old payloads (for retention policy)
*