Creationshop/cannaiq
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(tasks): Refactor task workflow with payload/refresh separation

Browse Source 
Major changes:
- Split crawl into payload_fetch (API → disk) and product_refresh (disk → DB)
- Add task chaining: store_discovery → product_discovery → payload_fetch → product_refresh
- Add payload storage utilities for gzipped JSON on filesystem
- Add /api/payloads endpoints for payload access and diffing
- Add DB-driven TaskScheduler with schedule persistence
- Track newDispensaryIds through discovery promotion for chaining
- Add stealth improvements: HTTP fingerprinting, proxy rotation enhancements
- Add Workers dashboard K8s scaling controls

New files:
- src/tasks/handlers/payload-fetch.ts - Fetches from API, saves to disk
- src/services/task-scheduler.ts - DB-driven schedule management
- src/utils/payload-storage.ts - Payload save/load utilities
- src/routes/payloads.ts - Payload API endpoints
- src/services/http-fingerprint.ts - Browser fingerprint generation
- docs/TASK_WORKFLOW_2024-12-10.md - Complete workflow documentation

Migrations:
- 078: Proxy consecutive 403 tracking
- 079: task_schedules table
- 080: raw_crawl_payloads table
- 081: payload column and last_fetch_at

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
Kelly
2025-12-10 22:15:35 -07:00
parent 1fb0eb94c2
commit 4949b22457
33 changed files with 4064 additions and 737 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
backend/src/discovery/discovery-crawler.ts
View File

@@ -172,6 +172,9 @@ export async function runFullDiscovery(
console.log(`Errors: ${totalErrors}`);
}
// Per TASK_WORKFLOW_2024-12-10.md: Track new dispensary IDs for task chaining
let newDispensaryIds: number[] = [];
// Step 4: Auto-validate and promote discovered locations
if (!dryRun && totalLocationsUpserted > 0) {
console.log('\n[Discovery] Step 4: Auto-promoting discovered locations...');
@@ -180,6 +183,13 @@ export async function runFullDiscovery(
console.log(` Created: ${promotionResult.created} new dispensaries`);
console.log(` Updated: ${promotionResult.updated} existing dispensaries`);
console.log(` Rejected: ${promotionResult.rejected} (validation failed)`);
// Per TASK_WORKFLOW_2024-12-10.md: Capture new IDs for task chaining
newDispensaryIds = promotionResult.newDispensaryIds;
if (newDispensaryIds.length > 0) {
console.log(` New store IDs for crawl: [${newDispensaryIds.join(', ')}]`);
}
if (promotionResult.rejectedRecords.length > 0) {
console.log(` Rejection reasons:`);
promotionResult.rejectedRecords.slice(0, 5).forEach(r => {
@@ -214,6 +224,8 @@ export async function runFullDiscovery(
totalLocationsFound,
totalLocationsUpserted,
durationMs,
// Per TASK_WORKFLOW_2024-12-10.md: Return new IDs for task chaining
newDispensaryIds,
};
}

8
backend/src/discovery/promotion.ts
View File

@@ -127,6 +127,8 @@ export interface PromotionSummary {
errors: string[];
}>;
durationMs: number;
// Per TASK_WORKFLOW_2024-12-10.md: Track new dispensary IDs for task chaining
newDispensaryIds: number[];
}
/**
@@ -469,6 +471,8 @@ export async function promoteDiscoveredLocations(
const results: PromotionResult[] = [];
const rejectedRecords: PromotionSummary['rejectedRecords'] = [];
// Per TASK_WORKFLOW_2024-12-10.md: Track new dispensary IDs for task chaining
const newDispensaryIds: number[] = [];
let created = 0;
let updated = 0;
let skipped = 0;
@@ -525,6 +529,8 @@ export async function promoteDiscoveredLocations(
if (promotionResult.action === 'created') {
created++;
// Per TASK_WORKFLOW_2024-12-10.md: Track new IDs for task chaining
newDispensaryIds.push(promotionResult.dispensaryId);
} else {
updated++;
}
@@ -548,6 +554,8 @@ export async function promoteDiscoveredLocations(
results,
rejectedRecords,
durationMs: Date.now() - startTime,
// Per TASK_WORKFLOW_2024-12-10.md: Return new IDs for task chaining
newDispensaryIds,
};
}

2
backend/src/discovery/types.ts
View File

@@ -211,6 +211,8 @@ export interface FullDiscoveryResult {
totalLocationsFound: number;
totalLocationsUpserted: number;
durationMs: number;
// Per TASK_WORKFLOW_2024-12-10.md: Track new dispensary IDs for task chaining
newDispensaryIds?: number[];
}
// ============================================================

19
backend/src/index.ts
View File

@@ -6,6 +6,8 @@ import { initializeMinio, isMinioEnabled } from './utils/minio';
import { initializeImageStorage } from './utils/image-storage';
import { logger } from './services/logger';
import { cleanupOrphanedJobs } from './services/proxyTestQueue';
// Per TASK_WORKFLOW_2024-12-10.md: Database-driven task scheduler
import { taskScheduler } from './services/task-scheduler';
import { runAutoMigrations } from './db/auto-migrate';
import { getPool } from './db/pool';
import healthRoutes from './routes/health';
@@ -142,6 +144,8 @@ import seoRoutes from './routes/seo';
import priceAnalyticsRoutes from './routes/price-analytics';
import tasksRoutes from './routes/tasks';
import workerRegistryRoutes from './routes/worker-registry';
// Per TASK_WORKFLOW_2024-12-10.md: Raw payload access API
import payloadsRoutes from './routes/payloads';
// Mark requests from trusted domains (cannaiq.co, findagram.co, findadispo.com)
// These domains can access the API without authentication
@@ -222,6 +226,10 @@ console.log('[Tasks] Routes registered at /api/tasks');
app.use('/api/worker-registry', workerRegistryRoutes);
console.log('[WorkerRegistry] Routes registered at /api/worker-registry');
// Per TASK_WORKFLOW_2024-12-10.md: Raw payload access API
app.use('/api/payloads', payloadsRoutes);
console.log('[Payloads] Routes registered at /api/payloads');
// Phase 3: Analytics V2 - Enhanced analytics with rec/med state segmentation
try {
const analyticsV2Router = createAnalyticsV2Router(getPool());
@@ -326,6 +334,17 @@ async function startServer() {
// Clean up any orphaned proxy test jobs from previous server runs
await cleanupOrphanedJobs();
// Per TASK_WORKFLOW_2024-12-10.md: Start database-driven task scheduler
// This replaces node-cron - schedules are stored in DB and survive restarts
// Uses SELECT FOR UPDATE SKIP LOCKED for multi-replica safety
try {
await taskScheduler.start();
logger.info('system', 'Task scheduler started');
} catch (err: any) {
// Non-fatal - scheduler can recover on next poll
logger.warn('system', `Task scheduler startup warning: ${err.message}`);
}
app.listen(PORT, () => {
logger.info('system', `Server running on port ${PORT}`);
console.log(`🚀 Server running on port ${PORT}`);

456
backend/src/platforms/dutchie/client.ts
View File

@@ -5,22 +5,35 @@
*
* DO NOT MODIFY THIS FILE WITHOUT EXPLICIT AUTHORIZATION.
*
* This is the canonical HTTP client for all Dutchie communication.
* All Dutchie workers (Alice, Bella, etc.) MUST use this client.
* Updated: 2025-12-10 per workflow-12102025.md
*
* KEY BEHAVIORS (per workflow-12102025.md):
* 1. startSession() gets identity from PROXY LOCATION, not task params
* 2. On 403: immediately get new IP + new fingerprint, then retry
* 3. After 3 consecutive 403s on same proxy → disable it (burned)
* 4. Language is always English (en-US)
*
* IMPLEMENTATION:
* - Uses curl via child_process.execSync (bypasses TLS fingerprinting)
* - NO Puppeteer, NO axios, NO fetch
* - Fingerprint rotation on 403
* - Uses intoli/user-agents via CrawlRotator for realistic fingerprints
* - Residential IP compatible
*
* USAGE:
* import { curlPost, curlGet, executeGraphQL } from '@dutchie/client';
* import { curlPost, curlGet, executeGraphQL, startSession } from '@dutchie/client';
*
* ============================================================
*/
import { execSync } from 'child_process';
import {
buildOrderedHeaders,
buildRefererFromMenuUrl,
getCurlBinary,
isCurlImpersonateAvailable,
HeaderContext,
BrowserType,
} from '../../services/http-fingerprint';
// ============================================================
// TYPES
@@ -32,6 +45,8 @@ export interface CurlResponse {
error?: string;
}
// Per workflow-12102025.md: fingerprint comes from CrawlRotator's BrowserFingerprint
// We keep a simplified interface here for header building
export interface Fingerprint {
userAgent: string;
acceptLanguage: string;
@@ -57,15 +72,13 @@ export const DUTCHIE_CONFIG = {
// ============================================================
// PROXY SUPPORT
// ============================================================
// Integrates with the CrawlRotator system from proxy-rotator.ts
// On 403 errors:
// 1. Record failure on current proxy
// 2. Rotate to next proxy
// 3. Retry with new proxy
// Per workflow-12102025.md:
// - On 403: recordBlock() → increment consecutive_403_count
// - After 3 consecutive 403s → proxy disabled
// - Immediately rotate to new IP + new fingerprint on 403
// ============================================================
import type { CrawlRotator, Proxy } from '../../services/crawl-rotator';
import type { CrawlRotator, BrowserFingerprint } from '../../services/crawl-rotator';
let currentProxy: string | null = null;
let crawlRotator: CrawlRotator | null = null;
@@ -92,13 +105,12 @@ export function getProxy(): string | null {
/**
* Set CrawlRotator for proxy rotation on 403s
* This enables automatic proxy rotation when blocked
* Per workflow-12102025.md: enables automatic rotation when blocked
*/
export function setCrawlRotator(rotator: CrawlRotator | null): void {
crawlRotator = rotator;
if (rotator) {
console.log('[Dutchie Client] CrawlRotator attached - proxy rotation enabled');
// Set initial proxy from rotator
const proxy = rotator.proxy.getCurrent();
if (proxy) {
currentProxy = rotator.proxy.getProxyUrl(proxy);
@@ -115,30 +127,41 @@ export function getCrawlRotator(): CrawlRotator | null {
}
/**
* Rotate to next proxy (called on 403)
* Handle 403 block - per workflow-12102025.md:
* 1. Record block on current proxy (increments consecutive_403_count)
* 2. Immediately rotate to new proxy (new IP)
* 3. Rotate fingerprint
* Returns false if no more proxies available
*/
async function rotateProxyOn403(error?: string): Promise<boolean> {
async function handle403Block(): Promise<boolean> {
if (!crawlRotator) {
console.warn('[Dutchie Client] No CrawlRotator - cannot handle 403');
return false;
}
// Record failure on current proxy
await crawlRotator.recordFailure(error || '403 Forbidden');
// Per workflow-12102025.md: record block (tracks consecutive 403s)
const wasDisabled = await crawlRotator.recordBlock();
if (wasDisabled) {
console.log('[Dutchie Client] Current proxy was disabled (3 consecutive 403s)');
}
// Per workflow-12102025.md: immediately get new IP + new fingerprint
const { proxy: nextProxy, fingerprint } = crawlRotator.rotateBoth();
// Rotate to next proxy
const nextProxy = crawlRotator.rotateProxy();
if (nextProxy) {
currentProxy = crawlRotator.proxy.getProxyUrl(nextProxy);
console.log(`[Dutchie Client] Rotated proxy: ${currentProxy.replace(/:[^:@]+@/, ':***@')}`);
console.log(`[Dutchie Client] Rotated to new proxy: ${currentProxy.replace(/:[^:@]+@/, ':***@')}`);
console.log(`[Dutchie Client] New fingerprint: ${fingerprint.userAgent.slice(0, 50)}...`);
return true;
}
console.warn('[Dutchie Client] No more proxies available');
console.error('[Dutchie Client] No more proxies available!');
return false;
}
/**
* Record success on current proxy
* Per workflow-12102025.md: resets consecutive_403_count
*/
async function recordProxySuccess(responseTimeMs?: number): Promise<void> {
if (crawlRotator) {
@@ -162,163 +185,69 @@ export const GRAPHQL_HASHES = {
GetAllCitiesByState: 'ae547a0466ace5a48f91e55bf6699eacd87e3a42841560f0c0eabed5a0a920e6',
};
// ============================================================
// FINGERPRINTS - Browser profiles for anti-detect
// ============================================================
const FINGERPRINTS: Fingerprint[] = [
// Chrome Windows (latest) - typical residential user, use first
{
userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
acceptLanguage: 'en-US,en;q=0.9',
secChUa: '"Google Chrome";v="131", "Chromium";v="131", "Not_A Brand";v="24"',
secChUaPlatform: '"Windows"',
secChUaMobile: '?0',
},
// Chrome Mac (latest)
{
userAgent: 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
acceptLanguage: 'en-US,en;q=0.9',
secChUa: '"Google Chrome";v="131", "Chromium";v="131", "Not_A Brand";v="24"',
secChUaPlatform: '"macOS"',
secChUaMobile: '?0',
},
// Chrome Windows (120)
{
userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
acceptLanguage: 'en-US,en;q=0.9',
secChUa: '"Chromium";v="120", "Google Chrome";v="120", "Not-A.Brand";v="99"',
secChUaPlatform: '"Windows"',
secChUaMobile: '?0',
},
// Firefox Windows
{
userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:133.0) Gecko/20100101 Firefox/133.0',
acceptLanguage: 'en-US,en;q=0.5',
},
// Safari Mac
{
userAgent: 'Mozilla/5.0 (Macintosh; Intel Mac OS X 14_2) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.2 Safari/605.1.15',
acceptLanguage: 'en-US,en;q=0.9',
},
// Edge Windows
{
userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36 Edg/131.0.0.0',
acceptLanguage: 'en-US,en;q=0.9',
secChUa: '"Microsoft Edge";v="131", "Chromium";v="131", "Not_A Brand";v="24"',
secChUaPlatform: '"Windows"',
secChUaMobile: '?0',
},
];
let currentFingerprintIndex = 0;
// Forward declaration for session (actual CrawlSession interface defined later)
let currentSession: {
sessionId: string;
fingerprint: Fingerprint;
proxyUrl: string | null;
stateCode?: string;
timezone?: string;
startedAt: Date;
} | null = null;
/**
* Get current fingerprint - returns session fingerprint if active, otherwise default
*/
export function getFingerprint(): Fingerprint {
// Use session fingerprint if a session is active
if (currentSession) {
return currentSession.fingerprint;
}
return FINGERPRINTS[currentFingerprintIndex];
}
export function rotateFingerprint(): Fingerprint {
currentFingerprintIndex = (currentFingerprintIndex + 1) % FINGERPRINTS.length;
const fp = FINGERPRINTS[currentFingerprintIndex];
console.log(`[Dutchie Client] Rotated to fingerprint: ${fp.userAgent.slice(0, 50)}...`);
return fp;
}
export function resetFingerprint(): void {
currentFingerprintIndex = 0;
}
/**
* Get a random fingerprint from the pool
*/
export function getRandomFingerprint(): Fingerprint {
const index = Math.floor(Math.random() * FINGERPRINTS.length);
return FINGERPRINTS[index];
}
// ============================================================
// SESSION MANAGEMENT
// Per-session fingerprint rotation for stealth
// Per workflow-12102025.md:
// - Session identity comes from PROXY LOCATION
// - NOT from task params (no stateCode/timezone params)
// - Language is always English
// ============================================================
export interface CrawlSession {
sessionId: string;
fingerprint: Fingerprint;
fingerprint: BrowserFingerprint;
proxyUrl: string | null;
stateCode?: string;
timezone?: string;
proxyTimezone?: string;
proxyState?: string;
startedAt: Date;
// Per workflow-12102025.md: Dynamic Referer per dispensary
menuUrl?: string;
referer: string;
}
// Note: currentSession variable declared earlier in file for proper scoping
let currentSession: CrawlSession | null = null;
/**
* Timezone to Accept-Language mapping
* US timezones all use en-US but this can be extended for international
* Start a new crawl session
*
* Per workflow-12102025.md:
* - NO state/timezone params - identity comes from proxy location
* - Gets fingerprint from CrawlRotator (uses intoli/user-agents)
* - Language is always English (en-US)
* - Dynamic Referer per dispensary (from menuUrl)
*
* @param menuUrl - The dispensary's menu URL for dynamic Referer header
*/
const TIMEZONE_TO_LOCALE: Record<string, string> = {
'America/Phoenix': 'en-US,en;q=0.9',
'America/Los_Angeles': 'en-US,en;q=0.9',
'America/Denver': 'en-US,en;q=0.9',
'America/Chicago': 'en-US,en;q=0.9',
'America/New_York': 'en-US,en;q=0.9',
'America/Detroit': 'en-US,en;q=0.9',
'America/Anchorage': 'en-US,en;q=0.9',
'Pacific/Honolulu': 'en-US,en;q=0.9',
};
export function startSession(menuUrl?: string): CrawlSession {
if (!crawlRotator) {
throw new Error('[Dutchie Client] Cannot start session without CrawlRotator');
}
/**
* Get Accept-Language header for a given timezone
*/
export function getLocaleForTimezone(timezone?: string): string {
if (!timezone) return 'en-US,en;q=0.9';
return TIMEZONE_TO_LOCALE[timezone] || 'en-US,en;q=0.9';
}
// Per workflow-12102025.md: get identity from proxy location
const proxyLocation = crawlRotator.getProxyLocation();
const fingerprint = crawlRotator.userAgent.getCurrent();
/**
* Start a new crawl session with a random fingerprint
* Call this before crawling a store to get a fresh identity
*/
export function startSession(stateCode?: string, timezone?: string): CrawlSession {
const baseFp = getRandomFingerprint();
// Override Accept-Language based on timezone for geographic consistency
const fingerprint: Fingerprint = {
...baseFp,
acceptLanguage: getLocaleForTimezone(timezone),
};
// Per workflow-12102025.md: Dynamic Referer per dispensary
const referer = buildRefererFromMenuUrl(menuUrl);
currentSession = {
sessionId: `session_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`,
fingerprint,
proxyUrl: currentProxy,
stateCode,
timezone,
proxyTimezone: proxyLocation?.timezone,
proxyState: proxyLocation?.state,
startedAt: new Date(),
menuUrl,
referer,
};
console.log(`[Dutchie Client] Started session ${currentSession.sessionId}`);
console.log(`[Dutchie Client] Fingerprint: ${fingerprint.userAgent.slice(0, 50)}...`);
console.log(`[Dutchie Client] Accept-Language: ${fingerprint.acceptLanguage}`);
if (timezone) {
console.log(`[Dutchie Client] Timezone: ${timezone}`);
console.log(`[Dutchie Client] Browser: ${fingerprint.browserName} (${fingerprint.deviceCategory})`);
console.log(`[Dutchie Client] DNT: ${fingerprint.httpFingerprint.hasDNT ? 'enabled' : 'disabled'}`);
console.log(`[Dutchie Client] TLS: ${fingerprint.httpFingerprint.curlImpersonateBinary}`);
console.log(`[Dutchie Client] Referer: ${referer}`);
if (proxyLocation?.timezone) {
console.log(`[Dutchie Client] Proxy: ${proxyLocation.state || 'unknown'} (${proxyLocation.timezone})`);
}
return currentSession;
@@ -347,48 +276,80 @@ export function getCurrentSession(): CrawlSession | null {
// ============================================================
/**
* Build headers for Dutchie requests
* Per workflow-12102025.md: Build headers using HTTP fingerprint system
* Returns headers in browser-specific order with all natural variations
*/
export function buildHeaders(refererPath: string, fingerprint?: Fingerprint): Record<string, string> {
const fp = fingerprint || getFingerprint();
const refererUrl = `https://dutchie.com${refererPath}`;
const headers: Record<string, string> = {
'accept': 'application/json, text/plain, */*',
'accept-language': fp.acceptLanguage,
'content-type': 'application/json',
'origin': 'https://dutchie.com',
'referer': refererUrl,
'user-agent': fp.userAgent,
'apollographql-client-name': 'Marketplace (production)',
};
if (fp.secChUa) {
headers['sec-ch-ua'] = fp.secChUa;
headers['sec-ch-ua-mobile'] = fp.secChUaMobile || '?0';
headers['sec-ch-ua-platform'] = fp.secChUaPlatform || '"Windows"';
headers['sec-fetch-dest'] = 'empty';
headers['sec-fetch-mode'] = 'cors';
headers['sec-fetch-site'] = 'same-site';
export function buildHeaders(isPost: boolean, contentLength?: number): { headers: Record<string, string>; orderedHeaders: string[] } {
if (!currentSession || !crawlRotator) {
throw new Error('[Dutchie Client] Cannot build headers without active session');
}
return headers;
const fp = currentSession.fingerprint;
const httpFp = fp.httpFingerprint;
// Per workflow-12102025.md: Build context for ordered headers
const context: HeaderContext = {
userAgent: fp.userAgent,
secChUa: fp.secChUa,
secChUaPlatform: fp.secChUaPlatform,
secChUaMobile: fp.secChUaMobile,
referer: currentSession.referer,
isPost,
contentLength,
};
// Per workflow-12102025.md: Get ordered headers from HTTP fingerprint service
return buildOrderedHeaders(httpFp, context);
}
/**
* Execute HTTP POST using curl (bypasses TLS fingerprinting)
* Per workflow-12102025.md: Get curl binary for current session's browser
* Uses curl-impersonate for TLS fingerprint matching
*/
export function curlPost(url: string, body: any, headers: Record<string, string>, timeout = 30000): CurlResponse {
const filteredHeaders = Object.entries(headers)
.filter(([k]) => k.toLowerCase() !== 'accept-encoding')
.map(([k, v]) => `-H '${k}: ${v}'`)
function getCurlBinaryForSession(): string {
if (!currentSession) {
return 'curl'; // Fallback to standard curl
}
const browserType = currentSession.fingerprint.browserName as BrowserType;
// Per workflow-12102025.md: Check if curl-impersonate is available
if (isCurlImpersonateAvailable(browserType)) {
return getCurlBinary(browserType);
}
// Fallback to standard curl with warning
console.warn(`[Dutchie Client] curl-impersonate not available for ${browserType}, using standard curl`);
return 'curl';
}
/**
* Per workflow-12102025.md: Execute HTTP POST using curl/curl-impersonate
* - Uses browser-specific TLS fingerprint via curl-impersonate
* - Headers sent in browser-specific order
* - Dynamic Referer per dispensary
*/
export function curlPost(url: string, body: any, timeout = 30000): CurlResponse {
const bodyJson = JSON.stringify(body);
// Per workflow-12102025.md: Build ordered headers for POST request
const { headers, orderedHeaders } = buildHeaders(true, bodyJson.length);
// Per workflow-12102025.md: Build header args in browser-specific order
const headerArgs = orderedHeaders
.filter(h => h !== 'Host' && h !== 'Content-Length') // curl handles these
.map(h => `-H '${h}: ${headers[h]}'`)
.join(' ');
const bodyJson = JSON.stringify(body).replace(/'/g, "'\\''");
const bodyEscaped = bodyJson.replace(/'/g, "'\\''");
const timeoutSec = Math.ceil(timeout / 1000);
const separator = '___HTTP_STATUS___';
const proxyArg = getProxyArg();
const cmd = `curl -s --compressed ${proxyArg} -w '${separator}%{http_code}' --max-time ${timeoutSec} ${filteredHeaders} -d '${bodyJson}' '${url}'`;
// Per workflow-12102025.md: Use curl-impersonate for TLS fingerprint matching
const curlBinary = getCurlBinaryForSession();
const cmd = `${curlBinary} -s --compressed ${proxyArg} -w '${separator}%{http_code}' --max-time ${timeoutSec} ${headerArgs} -d '${bodyEscaped}' '${url}'`;
try {
const output = execSync(cmd, {
@@ -427,19 +388,29 @@ export function curlPost(url: string, body: any, headers: Record<string, string>
}
/**
* Execute HTTP GET using curl (bypasses TLS fingerprinting)
* Returns HTML or JSON depending on response content-type
* Per workflow-12102025.md: Execute HTTP GET using curl/curl-impersonate
* - Uses browser-specific TLS fingerprint via curl-impersonate
* - Headers sent in browser-specific order
* - Dynamic Referer per dispensary
*/
export function curlGet(url: string, headers: Record<string, string>, timeout = 30000): CurlResponse {
const filteredHeaders = Object.entries(headers)
.filter(([k]) => k.toLowerCase() !== 'accept-encoding')
.map(([k, v]) => `-H '${k}: ${v}'`)
export function curlGet(url: string, timeout = 30000): CurlResponse {
// Per workflow-12102025.md: Build ordered headers for GET request
const { headers, orderedHeaders } = buildHeaders(false);
// Per workflow-12102025.md: Build header args in browser-specific order
const headerArgs = orderedHeaders
.filter(h => h !== 'Host' && h !== 'Content-Length') // curl handles these
.map(h => `-H '${h}: ${headers[h]}'`)
.join(' ');
const timeoutSec = Math.ceil(timeout / 1000);
const separator = '___HTTP_STATUS___';
const proxyArg = getProxyArg();
const cmd = `curl -s --compressed ${proxyArg} -w '${separator}%{http_code}' --max-time ${timeoutSec} ${filteredHeaders} '${url}'`;
// Per workflow-12102025.md: Use curl-impersonate for TLS fingerprint matching
const curlBinary = getCurlBinaryForSession();
const cmd = `${curlBinary} -s --compressed ${proxyArg} -w '${separator}%{http_code}' --max-time ${timeoutSec} ${headerArgs} '${url}'`;
try {
const output = execSync(cmd, {
@@ -459,7 +430,6 @@ export function curlGet(url: string, headers: Record<string, string>, timeout =
const responseBody = output.slice(0, separatorIndex);
const statusCode = parseInt(output.slice(separatorIndex + separator.length).trim(), 10);
// Try to parse as JSON, otherwise return as string (HTML)
try {
return { status: statusCode, data: JSON.parse(responseBody) };
} catch {
@@ -476,16 +446,22 @@ export function curlGet(url: string, headers: Record<string, string>, timeout =
// ============================================================
// GRAPHQL EXECUTION
// Per workflow-12102025.md:
// - On 403: immediately rotate IP + fingerprint (no delay first)
// - Then retry
// ============================================================
export interface ExecuteGraphQLOptions {
maxRetries?: number;
retryOn403?: boolean;
cName?: string; // Optional - used for Referer header, defaults to 'cities'
cName?: string;
}
/**
* Execute GraphQL query with curl (bypasses TLS fingerprinting)
* Per workflow-12102025.md: Execute GraphQL query with curl/curl-impersonate
* - Uses browser-specific TLS fingerprint
* - Headers in browser-specific order
* - On 403: immediately rotate IP + fingerprint, then retry
*/
export async function executeGraphQL(
operationName: string,
@@ -493,7 +469,12 @@ export async function executeGraphQL(
hash: string,
options: ExecuteGraphQLOptions
): Promise<any> {
const { maxRetries = 3, retryOn403 = true, cName = 'cities' } = options;
const { maxRetries = 3, retryOn403 = true } = options;
// Per workflow-12102025.md: Session must be active for requests
if (!currentSession) {
throw new Error('[Dutchie Client] Cannot execute GraphQL without active session - call startSession() first');
}
const body = {
operationName,
@@ -507,14 +488,14 @@ export async function executeGraphQL(
let attempt = 0;
while (attempt <= maxRetries) {
const fingerprint = getFingerprint();
const headers = buildHeaders(`/embedded-menu/${cName}`, fingerprint);
console.log(`[Dutchie Client] curl POST ${operationName} (attempt ${attempt + 1}/${maxRetries + 1})`);
const response = curlPost(DUTCHIE_CONFIG.graphqlEndpoint, body, headers, DUTCHIE_CONFIG.timeout);
const startTime = Date.now();
// Per workflow-12102025.md: curlPost now uses ordered headers and curl-impersonate
const response = curlPost(DUTCHIE_CONFIG.graphqlEndpoint, body, DUTCHIE_CONFIG.timeout);
const responseTime = Date.now() - startTime;
console.log(`[Dutchie Client] Response status: ${response.status}`);
console.log(`[Dutchie Client] Response status: ${response.status} (${responseTime}ms)`);
if (response.error) {
console.error(`[Dutchie Client] curl error: ${response.error}`);
@@ -527,6 +508,9 @@ export async function executeGraphQL(
}
if (response.status === 200) {
// Per workflow-12102025.md: success resets consecutive 403 count
await recordProxySuccess(responseTime);
if (response.data?.errors?.length > 0) {
console.warn(`[Dutchie Client] GraphQL errors: ${JSON.stringify(response.data.errors[0])}`);
}
@@ -534,11 +518,20 @@ export async function executeGraphQL(
}
if (response.status === 403 && retryOn403) {
console.warn(`[Dutchie Client] 403 blocked - rotating proxy and fingerprint...`);
await rotateProxyOn403('403 Forbidden on GraphQL');
rotateFingerprint();
// Per workflow-12102025.md: immediately rotate IP + fingerprint
console.warn(`[Dutchie Client] 403 blocked - immediately rotating proxy + fingerprint...`);
const hasMoreProxies = await handle403Block();
if (!hasMoreProxies) {
throw new Error('All proxies exhausted - no more IPs available');
}
// Per workflow-12102025.md: Update session referer after rotation
currentSession.referer = buildRefererFromMenuUrl(currentSession.menuUrl);
attempt++;
await sleep(1000 * attempt);
// Per workflow-12102025.md: small backoff after rotation
await sleep(500);
continue;
}
@@ -567,8 +560,10 @@ export interface FetchPageOptions {
}
/**
* Fetch HTML page from Dutchie (for city pages, dispensary pages, etc.)
* Returns raw HTML string
* Per workflow-12102025.md: Fetch HTML page from Dutchie
* - Uses browser-specific TLS fingerprint
* - Headers in browser-specific order
* - Same 403 handling as GraphQL
*/
export async function fetchPage(
path: string,
@@ -577,32 +572,22 @@ export async function fetchPage(
const { maxRetries = 3, retryOn403 = true } = options;
const url = `${DUTCHIE_CONFIG.baseUrl}${path}`;
// Per workflow-12102025.md: Session must be active for requests
if (!currentSession) {
throw new Error('[Dutchie Client] Cannot fetch page without active session - call startSession() first');
}
let attempt = 0;
while (attempt <= maxRetries) {
const fingerprint = getFingerprint();
const headers: Record<string, string> = {
'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8',
'accept-language': fingerprint.acceptLanguage,
'user-agent': fingerprint.userAgent,
};
if (fingerprint.secChUa) {
headers['sec-ch-ua'] = fingerprint.secChUa;
headers['sec-ch-ua-mobile'] = fingerprint.secChUaMobile || '?0';
headers['sec-ch-ua-platform'] = fingerprint.secChUaPlatform || '"Windows"';
headers['sec-fetch-dest'] = 'document';
headers['sec-fetch-mode'] = 'navigate';
headers['sec-fetch-site'] = 'none';
headers['sec-fetch-user'] = '?1';
headers['upgrade-insecure-requests'] = '1';
}
// Per workflow-12102025.md: curlGet now uses ordered headers and curl-impersonate
console.log(`[Dutchie Client] curl GET ${path} (attempt ${attempt + 1}/${maxRetries + 1})`);
const response = curlGet(url, headers, DUTCHIE_CONFIG.timeout);
const startTime = Date.now();
const response = curlGet(url, DUTCHIE_CONFIG.timeout);
const responseTime = Date.now() - startTime;
console.log(`[Dutchie Client] Response status: ${response.status}`);
console.log(`[Dutchie Client] Response status: ${response.status} (${responseTime}ms)`);
if (response.error) {
console.error(`[Dutchie Client] curl error: ${response.error}`);
@@ -614,15 +599,26 @@ export async function fetchPage(
}
if (response.status === 200) {
// Per workflow-12102025.md: success resets consecutive 403 count
await recordProxySuccess(responseTime);
return { html: response.data, status: response.status };
}
if (response.status === 403 && retryOn403) {
console.warn(`[Dutchie Client] 403 blocked - rotating proxy and fingerprint...`);
await rotateProxyOn403('403 Forbidden on page fetch');
rotateFingerprint();
// Per workflow-12102025.md: immediately rotate IP + fingerprint
console.warn(`[Dutchie Client] 403 blocked - immediately rotating proxy + fingerprint...`);
const hasMoreProxies = await handle403Block();
if (!hasMoreProxies) {
throw new Error('All proxies exhausted - no more IPs available');
}
// Per workflow-12102025.md: Update session after rotation
currentSession.referer = buildRefererFromMenuUrl(currentSession.menuUrl);
attempt++;
await sleep(1000 * attempt);
// Per workflow-12102025.md: small backoff after rotation
await sleep(500);
continue;
}

11
backend/src/platforms/dutchie/index.ts
View File

@@ -6,22 +6,17 @@
*/
export {
// HTTP Client
// HTTP Client (per workflow-12102025.md: uses curl-impersonate + ordered headers)
curlPost,
curlGet,
executeGraphQL,
fetchPage,
extractNextData,
// Headers & Fingerprints
// Headers (per workflow-12102025.md: browser-specific ordering)
buildHeaders,
getFingerprint,
rotateFingerprint,
resetFingerprint,
getRandomFingerprint,
getLocaleForTimezone,
// Session Management (per-store fingerprint rotation)
// Session Management (per workflow-12102025.md: menuUrl for dynamic Referer)
startSession,
endSession,
getCurrentSession,

173
backend/src/routes/job-queue.ts
View File

@@ -543,6 +543,9 @@ router.post('/bulk-priority', async (req: Request, res: Response) => {
/**
* POST /api/job-queue/enqueue - Add a new job to the queue
*
* 2024-12-10: Rewired to use worker_tasks via taskService.
* Legacy dispensary_crawl_jobs code commented out below.
*/
router.post('/enqueue', async (req: Request, res: Response) => {
try {
@@ -552,6 +555,59 @@ router.post('/enqueue', async (req: Request, res: Response) => {
return res.status(400).json({ success: false, error: 'dispensary_id is required' });
}
// 2024-12-10: Map legacy job_type to new task role
const roleMap: Record<string, string> = {
'dutchie_product_crawl': 'product_refresh',
'menu_detection': 'entry_point_discovery',
'menu_detection_single': 'entry_point_discovery',
'product_discovery': 'product_discovery',
'store_discovery': 'store_discovery',
};
const role = roleMap[job_type] || 'product_refresh';
// 2024-12-10: Use taskService to create task in worker_tasks table
const { taskService } = await import('../tasks/task-service');
// Check if task already pending for this dispensary
const existingTasks = await taskService.listTasks({
dispensary_id,
role: role as any,
status: ['pending', 'claimed', 'running'],
limit: 1,
});
if (existingTasks.length > 0) {
return res.json({
success: true,
task_id: existingTasks[0].id,
message: 'Task already queued'
});
}
const task = await taskService.createTask({
role: role as any,
dispensary_id,
priority,
});
res.json({ success: true, task_id: task.id, message: 'Task enqueued' });
} catch (error: any) {
console.error('[JobQueue] Error enqueuing task:', error);
res.status(500).json({ success: false, error: error.message });
}
});
/*
* LEGACY CODE - 2024-12-10: Commented out, was using orphaned dispensary_crawl_jobs table
*
router.post('/enqueue', async (req: Request, res: Response) => {
try {
const { dispensary_id, job_type = 'dutchie_product_crawl', priority = 0 } = req.body;
if (!dispensary_id) {
return res.status(400).json({ success: false, error: 'dispensary_id is required' });
}
// Check if job already pending for this dispensary
const existing = await pool.query(`
SELECT id FROM dispensary_crawl_jobs
@@ -585,6 +641,7 @@ router.post('/enqueue', async (req: Request, res: Response) => {
res.status(500).json({ success: false, error: error.message });
}
});
*/
/**
* POST /api/job-queue/pause - Pause queue processing
@@ -612,6 +669,8 @@ router.get('/paused', async (_req: Request, res: Response) => {
/**
* POST /api/job-queue/enqueue-batch - Queue multiple dispensaries at once
* Body: { dispensary_ids: number[], job_type?: string, priority?: number }
*
* 2024-12-10: Rewired to use worker_tasks via taskService.
*/
router.post('/enqueue-batch', async (req: Request, res: Response) => {
try {
@@ -625,35 +684,30 @@ router.post('/enqueue-batch', async (req: Request, res: Response) => {
return res.status(400).json({ success: false, error: 'Maximum 500 dispensaries per batch' });
}
// Insert jobs, skipping duplicates
const { rows } = await pool.query(`
INSERT INTO dispensary_crawl_jobs (dispensary_id, job_type, priority, trigger_type, status, created_at)
SELECT
d.id,
$2::text,
$3::integer,
'api_batch',
'pending',
NOW()
FROM dispensaries d
WHERE d.id = ANY($1::int[])
AND d.crawl_enabled = true
AND d.platform_dispensary_id IS NOT NULL
AND NOT EXISTS (
SELECT 1 FROM dispensary_crawl_jobs cj
WHERE cj.dispensary_id = d.id
AND cj.job_type = $2::text
AND cj.status IN ('pending', 'running')
)
RETURNING id, dispensary_id
`, [dispensary_ids, job_type, priority]);
// 2024-12-10: Map legacy job_type to new task role
const roleMap: Record<string, string> = {
'dutchie_product_crawl': 'product_refresh',
'menu_detection': 'entry_point_discovery',
'product_discovery': 'product_discovery',
};
const role = roleMap[job_type] || 'product_refresh';
// 2024-12-10: Use taskService to create tasks in worker_tasks table
const { taskService } = await import('../tasks/task-service');
const tasks = dispensary_ids.map(dispensary_id => ({
role: role as any,
dispensary_id,
priority,
}));
const createdCount = await taskService.createTasks(tasks);
res.json({
success: true,
queued: rows.length,
queued: createdCount,
requested: dispensary_ids.length,
job_ids: rows.map(r => r.id),
message: `Queued ${rows.length} of ${dispensary_ids.length} dispensaries`
message: `Queued ${createdCount} of ${dispensary_ids.length} dispensaries`
});
} catch (error: any) {
console.error('[JobQueue] Error batch enqueuing:', error);
@@ -664,6 +718,8 @@ router.post('/enqueue-batch', async (req: Request, res: Response) => {
/**
* POST /api/job-queue/enqueue-state - Queue all crawl-enabled dispensaries for a state
* Body: { state_code: string, job_type?: string, priority?: number, limit?: number }
*
* 2024-12-10: Rewired to use worker_tasks via taskService.
*/
router.post('/enqueue-state', async (req: Request, res: Response) => {
try {
@@ -673,52 +729,55 @@ router.post('/enqueue-state', async (req: Request, res: Response) => {
return res.status(400).json({ success: false, error: 'state_code is required (e.g., "AZ")' });
}
// Get state_id and queue jobs
const { rows } = await pool.query(`
WITH target_state AS (
SELECT id FROM states WHERE code = $1
)
INSERT INTO dispensary_crawl_jobs (dispensary_id, job_type, priority, trigger_type, status, created_at)
SELECT
d.id,
$2::text,
$3::integer,
'api_state',
'pending',
NOW()
FROM dispensaries d, target_state
WHERE d.state_id = target_state.id
// 2024-12-10: Map legacy job_type to new task role
const roleMap: Record<string, string> = {
'dutchie_product_crawl': 'product_refresh',
'menu_detection': 'entry_point_discovery',
'product_discovery': 'product_discovery',
};
const role = roleMap[job_type] || 'product_refresh';
// Get dispensary IDs for the state
const dispensaryResult = await pool.query(`
SELECT d.id
FROM dispensaries d
JOIN states s ON s.id = d.state_id
WHERE s.code = $1
AND d.crawl_enabled = true
AND d.platform_dispensary_id IS NOT NULL
AND NOT EXISTS (
SELECT 1 FROM dispensary_crawl_jobs cj
WHERE cj.dispensary_id = d.id
AND cj.job_type = $2::text
AND cj.status IN ('pending', 'running')
)
LIMIT $4::integer
RETURNING id, dispensary_id
`, [state_code.toUpperCase(), job_type, priority, limit]);
LIMIT $2
`, [state_code.toUpperCase(), limit]);
const dispensary_ids = dispensaryResult.rows.map((r: any) => r.id);
// 2024-12-10: Use taskService to create tasks in worker_tasks table
const { taskService } = await import('../tasks/task-service');
const tasks = dispensary_ids.map((dispensary_id: number) => ({
role: role as any,
dispensary_id,
priority,
}));
const createdCount = await taskService.createTasks(tasks);
// Get total available count
const countResult = await pool.query(`
WITH target_state AS (
SELECT id FROM states WHERE code = $1
)
SELECT COUNT(*) as total
FROM dispensaries d, target_state
WHERE d.state_id = target_state.id
FROM dispensaries d
JOIN states s ON s.id = d.state_id
WHERE s.code = $1
AND d.crawl_enabled = true
AND d.platform_dispensary_id IS NOT NULL
`, [state_code.toUpperCase()]);
res.json({
success: true,
queued: rows.length,
queued: createdCount,
total_available: parseInt(countResult.rows[0].total),
state: state_code.toUpperCase(),
job_type,
message: `Queued ${rows.length} dispensaries for ${state_code.toUpperCase()}`
role,
message: `Queued ${createdCount} dispensaries for ${state_code.toUpperCase()}`
});
} catch (error: any) {
console.error('[JobQueue] Error enqueuing state:', error);

334
backend/src/routes/payloads.ts Normal file
View File

@@ -0,0 +1,334 @@
/**
* Payload Routes
*
* Per TASK_WORKFLOW_2024-12-10.md: API access to raw crawl payloads.
*
* Endpoints:
* - GET /api/payloads - List payload metadata (paginated)
* - GET /api/payloads/:id - Get payload metadata by ID
* - GET /api/payloads/:id/data - Get full payload JSON
* - GET /api/payloads/store/:dispensaryId - List payloads for a store
* - GET /api/payloads/store/:dispensaryId/latest - Get latest payload for a store
* - GET /api/payloads/store/:dispensaryId/diff - Diff two payloads
*/
import { Router, Request, Response } from 'express';
import { getPool } from '../db/pool';
import {
loadRawPayloadById,
getLatestPayload,
getRecentPayloads,
listPayloadMetadata,
} from '../utils/payload-storage';
import { Pool } from 'pg';
const router = Router();
// Get pool instance for queries
const getDbPool = (): Pool => getPool() as unknown as Pool;
/**
* GET /api/payloads
* List payload metadata (paginated)
*/
router.get('/', async (req: Request, res: Response) => {
try {
const pool = getDbPool();
const limit = Math.min(parseInt(req.query.limit as string) || 50, 100);
const offset = parseInt(req.query.offset as string) || 0;
const dispensaryId = req.query.dispensary_id ? parseInt(req.query.dispensary_id as string) : undefined;
const payloads = await listPayloadMetadata(pool, {
dispensaryId,
limit,
offset,
});
res.json({
success: true,
payloads,
pagination: { limit, offset },
});
} catch (error: any) {
console.error('[Payloads] List error:', error.message);
res.status(500).json({ success: false, error: error.message });
}
});
/**
* GET /api/payloads/:id
* Get payload metadata by ID
*/
router.get('/:id', async (req: Request, res: Response) => {
try {
const pool = getDbPool();
const id = parseInt(req.params.id);
const result = await pool.query(`
SELECT
p.id,
p.dispensary_id,
p.crawl_run_id,
p.storage_path,
p.product_count,
p.size_bytes,
p.size_bytes_raw,
p.fetched_at,
p.processed_at,
p.checksum_sha256,
d.name as dispensary_name
FROM raw_crawl_payloads p
LEFT JOIN dispensaries d ON d.id = p.dispensary_id
WHERE p.id = $1
`, [id]);
if (result.rows.length === 0) {
return res.status(404).json({ success: false, error: 'Payload not found' });
}
res.json({
success: true,
payload: result.rows[0],
});
} catch (error: any) {
console.error('[Payloads] Get error:', error.message);
res.status(500).json({ success: false, error: error.message });
}
});
/**
* GET /api/payloads/:id/data
* Get full payload JSON (decompressed from disk)
*/
router.get('/:id/data', async (req: Request, res: Response) => {
try {
const pool = getDbPool();
const id = parseInt(req.params.id);
const result = await loadRawPayloadById(pool, id);
if (!result) {
return res.status(404).json({ success: false, error: 'Payload not found' });
}
res.json({
success: true,
metadata: result.metadata,
data: result.payload,
});
} catch (error: any) {
console.error('[Payloads] Get data error:', error.message);
res.status(500).json({ success: false, error: error.message });
}
});
/**
* GET /api/payloads/store/:dispensaryId
* List payloads for a specific store
*/
router.get('/store/:dispensaryId', async (req: Request, res: Response) => {
try {
const pool = getDbPool();
const dispensaryId = parseInt(req.params.dispensaryId);
const limit = Math.min(parseInt(req.query.limit as string) || 20, 100);
const offset = parseInt(req.query.offset as string) || 0;
const payloads = await listPayloadMetadata(pool, {
dispensaryId,
limit,
offset,
});
res.json({
success: true,
dispensaryId,
payloads,
pagination: { limit, offset },
});
} catch (error: any) {
console.error('[Payloads] Store list error:', error.message);
res.status(500).json({ success: false, error: error.message });
}
});
/**
* GET /api/payloads/store/:dispensaryId/latest
* Get the latest payload for a store (with full data)
*/
router.get('/store/:dispensaryId/latest', async (req: Request, res: Response) => {
try {
const pool = getDbPool();
const dispensaryId = parseInt(req.params.dispensaryId);
const result = await getLatestPayload(pool, dispensaryId);
if (!result) {
return res.status(404).json({
success: false,
error: `No payloads found for dispensary ${dispensaryId}`,
});
}
res.json({
success: true,
metadata: result.metadata,
data: result.payload,
});
} catch (error: any) {
console.error('[Payloads] Latest error:', error.message);
res.status(500).json({ success: false, error: error.message });
}
});
/**
* GET /api/payloads/store/:dispensaryId/diff
* Compare two payloads for a store
*
* Query params:
* - from: payload ID (older)
* - to: payload ID (newer) - optional, defaults to latest
*/
router.get('/store/:dispensaryId/diff', async (req: Request, res: Response) => {
try {
const pool = getDbPool();
const dispensaryId = parseInt(req.params.dispensaryId);
const fromId = req.query.from ? parseInt(req.query.from as string) : undefined;
const toId = req.query.to ? parseInt(req.query.to as string) : undefined;
let fromPayload: any;
let toPayload: any;
if (fromId && toId) {
// Load specific payloads
const [from, to] = await Promise.all([
loadRawPayloadById(pool, fromId),
loadRawPayloadById(pool, toId),
]);
fromPayload = from;
toPayload = to;
} else {
// Load two most recent
const recent = await getRecentPayloads(pool, dispensaryId, 2);
if (recent.length < 2) {
return res.status(400).json({
success: false,
error: 'Need at least 2 payloads to diff. Only found ' + recent.length,
});
}
toPayload = recent[0]; // Most recent
fromPayload = recent[1]; // Previous
}
if (!fromPayload || !toPayload) {
return res.status(404).json({ success: false, error: 'One or both payloads not found' });
}
// Build product maps by ID
const fromProducts = new Map<string, any>();
const toProducts = new Map<string, any>();
for (const p of fromPayload.payload.products || []) {
const id = p._id || p.id;
if (id) fromProducts.set(id, p);
}
for (const p of toPayload.payload.products || []) {
const id = p._id || p.id;
if (id) toProducts.set(id, p);
}
// Find differences
const added: any[] = [];
const removed: any[] = [];
const priceChanges: any[] = [];
const stockChanges: any[] = [];
// Products in "to" but not in "from" = added
for (const [id, product] of toProducts) {
if (!fromProducts.has(id)) {
added.push({
id,
name: product.name,
brand: product.brand?.name,
price: product.Prices?.[0]?.price,
});
}
}
// Products in "from" but not in "to" = removed
for (const [id, product] of fromProducts) {
if (!toProducts.has(id)) {
removed.push({
id,
name: product.name,
brand: product.brand?.name,
price: product.Prices?.[0]?.price,
});
}
}
// Products in both - check for changes
for (const [id, toProduct] of toProducts) {
const fromProduct = fromProducts.get(id);
if (!fromProduct) continue;
const fromPrice = fromProduct.Prices?.[0]?.price;
const toPrice = toProduct.Prices?.[0]?.price;
if (fromPrice !== toPrice) {
priceChanges.push({
id,
name: toProduct.name,
brand: toProduct.brand?.name,
oldPrice: fromPrice,
newPrice: toPrice,
change: toPrice && fromPrice ? toPrice - fromPrice : null,
});
}
const fromStock = fromProduct.Status || fromProduct.status;
const toStock = toProduct.Status || toProduct.status;
if (fromStock !== toStock) {
stockChanges.push({
id,
name: toProduct.name,
brand: toProduct.brand?.name,
oldStatus: fromStock,
newStatus: toStock,
});
}
}
res.json({
success: true,
from: {
id: fromPayload.metadata.id,
fetchedAt: fromPayload.metadata.fetchedAt,
productCount: fromPayload.metadata.productCount,
},
to: {
id: toPayload.metadata.id,
fetchedAt: toPayload.metadata.fetchedAt,
productCount: toPayload.metadata.productCount,
},
diff: {
added: added.length,
removed: removed.length,
priceChanges: priceChanges.length,
stockChanges: stockChanges.length,
},
details: {
added,
removed,
priceChanges,
stockChanges,
},
});
} catch (error: any) {
console.error('[Payloads] Diff error:', error.message);
res.status(500).json({ success: false, error: error.message });
}
});
export default router;

154
backend/src/routes/workers.ts
View File

@@ -17,13 +17,167 @@
* GET /api/monitor/jobs - Get recent job history
* GET /api/monitor/active-jobs - Get currently running jobs
* GET /api/monitor/summary - Get monitoring summary
*
* K8s Scaling (added 2024-12-10):
* GET /api/workers/k8s/replicas - Get current replica count
* POST /api/workers/k8s/scale - Scale worker replicas up/down
*/
import { Router, Request, Response } from 'express';
import { pool } from '../db/pool';
import * as k8s from '@kubernetes/client-node';
const router = Router();
// ============================================================
// K8S SCALING CONFIGURATION (added 2024-12-10)
// Per TASK_WORKFLOW_2024-12-10.md: Admin can scale workers from UI
// ============================================================
const K8S_NAMESPACE = process.env.K8S_NAMESPACE || 'dispensary-scraper';
const K8S_STATEFULSET_NAME = process.env.K8S_WORKER_STATEFULSET || 'scraper-worker';
// Initialize K8s client - uses in-cluster config when running in K8s,
// or kubeconfig when running locally
let k8sAppsApi: k8s.AppsV1Api | null = null;
function getK8sClient(): k8s.AppsV1Api | null {
if (k8sAppsApi) return k8sAppsApi;
try {
const kc = new k8s.KubeConfig();
// Try in-cluster config first (when running as a pod)
// Falls back to default kubeconfig (~/.kube/config) for local dev
try {
kc.loadFromCluster();
} catch {
kc.loadFromDefault();
}
k8sAppsApi = kc.makeApiClient(k8s.AppsV1Api);
return k8sAppsApi;
} catch (err: any) {
console.warn('[Workers] K8s client not available:', err.message);
return null;
}
}
// ============================================================
// K8S SCALING ROUTES (added 2024-12-10)
// Per TASK_WORKFLOW_2024-12-10.md: Admin can scale workers from UI
// ============================================================
/**
* GET /api/workers/k8s/replicas - Get current worker replica count
* Returns current and desired replica counts from the StatefulSet
*/
router.get('/k8s/replicas', async (_req: Request, res: Response) => {
const client = getK8sClient();
if (!client) {
return res.status(503).json({
success: false,
error: 'K8s client not available (not running in cluster or no kubeconfig)',
replicas: null,
});
}
try {
const response = await client.readNamespacedStatefulSet({
name: K8S_STATEFULSET_NAME,
namespace: K8S_NAMESPACE,
});
const statefulSet = response;
res.json({
success: true,
replicas: {
current: statefulSet.status?.readyReplicas || 0,
desired: statefulSet.spec?.replicas || 0,
available: statefulSet.status?.availableReplicas || 0,
updated: statefulSet.status?.updatedReplicas || 0,
},
statefulset: K8S_STATEFULSET_NAME,
namespace: K8S_NAMESPACE,
});
} catch (err: any) {
console.error('[Workers] K8s replicas error:', err.body?.message || err.message);
res.status(500).json({
success: false,
error: err.body?.message || err.message,
});
}
});
/**
* POST /api/workers/k8s/scale - Scale worker replicas
* Body: { replicas: number } - desired replica count (1-20)
*/
router.post('/k8s/scale', async (req: Request, res: Response) => {
const client = getK8sClient();
if (!client) {
return res.status(503).json({
success: false,
error: 'K8s client not available (not running in cluster or no kubeconfig)',
});
}
const { replicas } = req.body;
// Validate replica count
if (typeof replicas !== 'number' || replicas < 0 || replicas > 20) {
return res.status(400).json({
success: false,
error: 'replicas must be a number between 0 and 20',
});
}
try {
// Get current state first
const currentResponse = await client.readNamespacedStatefulSetScale({
name: K8S_STATEFULSET_NAME,
namespace: K8S_NAMESPACE,
});
const currentReplicas = currentResponse.spec?.replicas || 0;
// Update scale using replaceNamespacedStatefulSetScale
await client.replaceNamespacedStatefulSetScale({
name: K8S_STATEFULSET_NAME,
namespace: K8S_NAMESPACE,
body: {
apiVersion: 'autoscaling/v1',
kind: 'Scale',
metadata: {
name: K8S_STATEFULSET_NAME,
namespace: K8S_NAMESPACE,
},
spec: {
replicas: replicas,
},
},
});
console.log(`[Workers] Scaled ${K8S_STATEFULSET_NAME} from ${currentReplicas} to ${replicas} replicas`);
res.json({
success: true,
message: `Scaled from ${currentReplicas} to ${replicas} replicas`,
previous: currentReplicas,
desired: replicas,
statefulset: K8S_STATEFULSET_NAME,
namespace: K8S_NAMESPACE,
});
} catch (err: any) {
console.error('[Workers] K8s scale error:', err.body?.message || err.message);
res.status(500).json({
success: false,
error: err.body?.message || err.message,
});
}
});
// ============================================================
// STATIC ROUTES (must come before parameterized routes)
// ============================================================

19
backend/src/scripts/crawl-single-store.ts
View File

@@ -16,10 +16,11 @@ import {
executeGraphQL,
startSession,
endSession,
getFingerprint,
setCrawlRotator,
GRAPHQL_HASHES,
DUTCHIE_CONFIG,
} from '../platforms/dutchie';
import { CrawlRotator } from '../services/crawl-rotator';
dotenv.config();
@@ -108,19 +109,27 @@ async function main() {
// ============================================================
// STEP 2: Start stealth session
// Per workflow-12102025.md: Initialize CrawlRotator and start session with menuUrl
// ============================================================
console.log('┌─────────────────────────────────────────────────────────────┐');
console.log('│ STEP 2: Start Stealth Session │');
console.log('└─────────────────────────────────────────────────────────────┘');
// Use Arizona timezone for this store
const session = startSession(disp.state || 'AZ', 'America/Phoenix');
// Per workflow-12102025.md: Initialize CrawlRotator (required for sessions)
const rotator = new CrawlRotator();
setCrawlRotator(rotator);
const fp = getFingerprint();
// Per workflow-12102025.md: startSession takes menuUrl for dynamic Referer
const session = startSession(disp.menu_url);
const fp = session.fingerprint;
console.log(` Session ID: ${session.sessionId}`);
console.log(` Browser: ${fp.browserName} (${fp.deviceCategory})`);
console.log(` User-Agent: ${fp.userAgent.slice(0, 60)}...`);
console.log(` Accept-Language: ${fp.acceptLanguage}`);
console.log(` Sec-CH-UA: ${fp.secChUa || '(not set)'}`);
console.log(` Referer: ${session.referer}`);
console.log(` DNT: ${fp.httpFingerprint.hasDNT ? 'enabled' : 'disabled'}`);
console.log(` TLS: ${fp.httpFingerprint.curlImpersonateBinary}`);
console.log('');
// ============================================================

194
backend/src/scripts/test-stealth-session.ts
View File

@@ -1,10 +1,10 @@
/**
* Test script for stealth session management
*
* Tests:
* 1. Per-session fingerprint rotation
* 2. Geographic consistency (timezone → Accept-Language)
* 3. Proxy location loading from database
* Per workflow-12102025.md:
* - Tests HTTP fingerprinting (browser-specific headers + ordering)
* - Tests UA generation (device distribution, browser filtering)
* - Tests dynamic Referer per dispensary
*
* Usage:
* npx tsx src/scripts/test-stealth-session.ts
@@ -14,104 +14,142 @@ import {
startSession,
endSession,
getCurrentSession,
getFingerprint,
getRandomFingerprint,
getLocaleForTimezone,
buildHeaders,
setCrawlRotator,
} from '../platforms/dutchie';
import { CrawlRotator } from '../services/crawl-rotator';
import {
generateHTTPFingerprint,
buildRefererFromMenuUrl,
BrowserType,
} from '../services/http-fingerprint';
console.log('='.repeat(60));
console.log('STEALTH SESSION TEST');
console.log('STEALTH SESSION TEST (per workflow-12102025.md)');
console.log('='.repeat(60));
// Test 1: Timezone to Locale mapping
console.log('\n[Test 1] Timezone to Locale Mapping:');
const testTimezones = [
'America/Phoenix',
'America/Los_Angeles',
'America/New_York',
'America/Chicago',
// Initialize CrawlRotator (required for sessions)
console.log('\n[Setup] Initializing CrawlRotator...');
const rotator = new CrawlRotator();
setCrawlRotator(rotator);
console.log(' CrawlRotator initialized');
// Test 1: HTTP Fingerprint Generation
console.log('\n[Test 1] HTTP Fingerprint Generation:');
const browsers: BrowserType[] = ['Chrome', 'Firefox', 'Safari', 'Edge'];
for (const browser of browsers) {
const httpFp = generateHTTPFingerprint(browser);
console.log(` ${browser}:`);
console.log(` TLS binary: ${httpFp.curlImpersonateBinary}`);
console.log(` DNT: ${httpFp.hasDNT ? 'enabled' : 'disabled'}`);
console.log(` Header order: ${httpFp.headerOrder.slice(0, 5).join(', ')}...`);
}
// Test 2: Dynamic Referer from menu URLs
console.log('\n[Test 2] Dynamic Referer from Menu URLs:');
const testUrls = [
'https://dutchie.com/embedded-menu/harvest-of-tempe',
'https://dutchie.com/dispensary/zen-leaf-mesa',
'/embedded-menu/deeply-rooted',
'/dispensary/curaleaf-phoenix',
null,
undefined,
'Invalid/Timezone',
];
for (const tz of testTimezones) {
const locale = getLocaleForTimezone(tz);
console.log(` ${tz || '(undefined)'} → ${locale}`);
for (const url of testUrls) {
const referer = buildRefererFromMenuUrl(url);
console.log(` ${url || '(null/undefined)'}`);
console.log(` → ${referer}`);
}
// Test 2: Random fingerprint selection
console.log('\n[Test 2] Random Fingerprint Selection (5 samples):');
for (let i = 0; i < 5; i++) {
const fp = getRandomFingerprint();
console.log(` ${i + 1}. ${fp.userAgent.slice(0, 60)}...`);
// Test 3: Session with Dynamic Referer
console.log('\n[Test 3] Session with Dynamic Referer:');
const testMenuUrl = 'https://dutchie.com/dispensary/harvest-of-tempe';
console.log(` Starting session with menuUrl: ${testMenuUrl}`);
const session1 = startSession(testMenuUrl);
console.log(` Session ID: ${session1.sessionId}`);
console.log(` Browser: ${session1.fingerprint.browserName}`);
console.log(` Device: ${session1.fingerprint.deviceCategory}`);
console.log(` Referer: ${session1.referer}`);
console.log(` DNT: ${session1.fingerprint.httpFingerprint.hasDNT ? 'enabled' : 'disabled'}`);
console.log(` TLS: ${session1.fingerprint.httpFingerprint.curlImpersonateBinary}`);
// Test 4: Build Headers (browser-specific order)
console.log('\n[Test 4] Build Headers (browser-specific order):');
const { headers, orderedHeaders } = buildHeaders(true, 1000);
console.log(` Headers built for ${session1.fingerprint.browserName}:`);
console.log(` Order: ${orderedHeaders.join(' → ')}`);
console.log(` Sample headers:`);
console.log(` User-Agent: ${headers['User-Agent']?.slice(0, 50)}...`);
console.log(` Accept: ${headers['Accept']}`);
console.log(` Accept-Language: ${headers['Accept-Language']}`);
console.log(` Referer: ${headers['Referer']}`);
if (headers['sec-ch-ua']) {
console.log(` sec-ch-ua: ${headers['sec-ch-ua']}`);
}
if (headers['DNT']) {
console.log(` DNT: ${headers['DNT']}`);
}
// Test 3: Session Management
console.log('\n[Test 3] Session Management:');
// Before session - should use default fingerprint
console.log(' Before session:');
const beforeFp = getFingerprint();
console.log(` getFingerprint(): ${beforeFp.userAgent.slice(0, 50)}...`);
console.log(` getCurrentSession(): ${getCurrentSession()}`);
// Start session with Arizona timezone
console.log('\n Starting session (AZ, America/Phoenix):');
const session1 = startSession('AZ', 'America/Phoenix');
console.log(` Session ID: ${session1.sessionId}`);
console.log(` Fingerprint UA: ${session1.fingerprint.userAgent.slice(0, 50)}...`);
console.log(` Accept-Language: ${session1.fingerprint.acceptLanguage}`);
console.log(` Timezone: ${session1.timezone}`);
// During session - should use session fingerprint
console.log('\n During session:');
const duringFp = getFingerprint();
console.log(` getFingerprint(): ${duringFp.userAgent.slice(0, 50)}...`);
console.log(` Same as session? ${duringFp.userAgent === session1.fingerprint.userAgent}`);
// Test buildHeaders with session
console.log('\n buildHeaders() during session:');
const headers = buildHeaders('/embedded-menu/test-store');
console.log(` User-Agent: ${headers['user-agent'].slice(0, 50)}...`);
console.log(` Accept-Language: ${headers['accept-language']}`);
console.log(` Origin: ${headers['origin']}`);
console.log(` Referer: ${headers['referer']}`);
// End session
console.log('\n Ending session:');
endSession();
console.log(` getCurrentSession(): ${getCurrentSession()}`);
// Test 4: Multiple sessions should have different fingerprints
console.log('\n[Test 4] Multiple Sessions (fingerprint variety):');
const fingerprints: string[] = [];
// Test 5: Multiple Sessions (UA variety)
console.log('\n[Test 5] Multiple Sessions (UA & fingerprint variety):');
const sessions: {
browser: string;
device: string;
hasDNT: boolean;
}[] = [];
for (let i = 0; i < 10; i++) {
const session = startSession('CA', 'America/Los_Angeles');
fingerprints.push(session.fingerprint.userAgent);
const session = startSession(`/dispensary/store-${i}`);
sessions.push({
browser: session.fingerprint.browserName,
device: session.fingerprint.deviceCategory,
hasDNT: session.fingerprint.httpFingerprint.hasDNT,
});
endSession();
}
const uniqueCount = new Set(fingerprints).size;
console.log(` 10 sessions created, ${uniqueCount} unique fingerprints`);
console.log(` Variety: ${uniqueCount >= 3 ? '✅ Good' : '⚠️ Low - may need more fingerprint options'}`);
// Count distribution
const browserCounts: Record<string, number> = {};
const deviceCounts: Record<string, number> = {};
let dntCount = 0;
// Test 5: Geographic consistency check
console.log('\n[Test 5] Geographic Consistency:');
const geoTests = [
{ state: 'AZ', tz: 'America/Phoenix' },
{ state: 'CA', tz: 'America/Los_Angeles' },
{ state: 'NY', tz: 'America/New_York' },
{ state: 'IL', tz: 'America/Chicago' },
];
for (const s of sessions) {
browserCounts[s.browser] = (browserCounts[s.browser] || 0) + 1;
deviceCounts[s.device] = (deviceCounts[s.device] || 0) + 1;
if (s.hasDNT) dntCount++;
}
for (const { state, tz } of geoTests) {
const session = startSession(state, tz);
const consistent = session.fingerprint.acceptLanguage.includes('en-US');
console.log(` ${state} (${tz}): Accept-Language=${session.fingerprint.acceptLanguage} ${consistent ? '✅' : '❌'}`);
console.log(` 10 sessions created:`);
console.log(` Browsers: ${JSON.stringify(browserCounts)}`);
console.log(` Devices: ${JSON.stringify(deviceCounts)}`);
console.log(` DNT enabled: ${dntCount}/10 (expected ~30%)`);
// Test 6: Device distribution check (per workflow-12102025.md: 62/36/2)
console.log('\n[Test 6] Device Distribution (larger sample):');
const deviceSamples: string[] = [];
for (let i = 0; i < 100; i++) {
const session = startSession();
deviceSamples.push(session.fingerprint.deviceCategory);
endSession();
}
const mobileCount = deviceSamples.filter(d => d === 'mobile').length;
const desktopCount = deviceSamples.filter(d => d === 'desktop').length;
const tabletCount = deviceSamples.filter(d => d === 'tablet').length;
console.log(` 100 sessions (expected: 62% mobile, 36% desktop, 2% tablet):`);
console.log(` Mobile: ${mobileCount}%`);
console.log(` Desktop: ${desktopCount}%`);
console.log(` Tablet: ${tabletCount}%`);
console.log(` Distribution: ${Math.abs(mobileCount - 62) < 15 && Math.abs(desktopCount - 36) < 15 ? '✅ Reasonable' : '⚠️ Off target'}`);
console.log('\n' + '='.repeat(60));
console.log('TEST COMPLETE');
console.log('='.repeat(60));

438
backend/src/services/crawl-rotator.ts
View File

@@ -1,49 +1,53 @@
/**
* Crawl Rotator - Proxy & User Agent Rotation for Crawlers
*
* Manages rotation of proxies and user agents to avoid blocks.
* Used by platform-specific crawlers (Dutchie, Jane, etc.)
* Updated: 2025-12-10 per workflow-12102025.md
*
* KEY BEHAVIORS (per workflow-12102025.md):
* 1. Task determines WHAT work to do, proxy determines SESSION IDENTITY
* 2. Proxy location (timezone) sets Accept-Language headers (always English)
* 3. On 403: immediately get new IP, new fingerprint, retry
* 4. After 3 consecutive 403s on same proxy with different fingerprints → disable proxy
*
* USER-AGENT GENERATION (per workflow-12102025.md):
* - Device distribution: Mobile 62%, Desktop 36%, Tablet 2%
* - Browser whitelist: Chrome, Safari, Edge, Firefox only
* - UA sticks until IP rotates
* - Failure = alert admin + stop crawl (no fallback)
*
* Uses intoli/user-agents for realistic UA generation with daily-updated data.
*
* Canonical location: src/services/crawl-rotator.ts
*/
import { Pool } from 'pg';
import UserAgent from 'user-agents';
import {
HTTPFingerprint,
generateHTTPFingerprint,
BrowserType,
} from './http-fingerprint';
// ============================================================
// USER AGENT CONFIGURATION
// UA CONSTANTS (per workflow-12102025.md)
// ============================================================
/**
* Modern browser user agents (Chrome, Firefox, Safari, Edge on various platforms)
* Updated: 2024
* Per workflow-12102025.md: Device category distribution (hardcoded)
* Mobile: 62%, Desktop: 36%, Tablet: 2%
*/
export const USER_AGENTS = [
// Chrome on Windows
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36',
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36',
const DEVICE_WEIGHTS = {
mobile: 62,
desktop: 36,
tablet: 2,
} as const;
// Chrome on macOS
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36',
// Firefox on Windows
'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:121.0) Gecko/20100101 Firefox/121.0',
'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:120.0) Gecko/20100101 Firefox/120.0',
// Firefox on macOS
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:121.0) Gecko/20100101 Firefox/121.0',
// Safari on macOS
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.2 Safari/605.1.15',
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.1 Safari/605.1.15',
// Edge on Windows
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36 Edg/120.0.0.0',
// Chrome on Linux
'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
];
/**
* Per workflow-12102025.md: Browser whitelist
* Only Chrome (67%), Safari (20%), Edge (6%), Firefox (3%)
* Samsung Internet, Opera, and other niche browsers are filtered out
*/
const ALLOWED_BROWSERS = ['Chrome', 'Safari', 'Edge', 'Firefox'] as const;
// ============================================================
// PROXY TYPES
@@ -61,8 +65,13 @@ export interface Proxy {
failureCount: number;
successCount: number;
avgResponseTimeMs: number | null;
maxConnections: number; // Number of concurrent connections allowed (for rotating proxies)
// Location info (if known)
maxConnections: number;
/**
* Per workflow-12102025.md: Track consecutive 403s with different fingerprints.
* After 3 consecutive 403s → disable proxy (it's burned).
*/
consecutive403Count: number;
// Location info - determines session headers per workflow-12102025.md
city?: string;
state?: string;
country?: string;
@@ -77,6 +86,40 @@ export interface ProxyStats {
avgSuccessRate: number;
}
// ============================================================
// FINGERPRINT TYPE
// Per workflow-12102025.md: Full browser fingerprint from user-agents
// ============================================================
export interface BrowserFingerprint {
userAgent: string;
platform: string;
screenWidth: number;
screenHeight: number;
viewportWidth: number;
viewportHeight: number;
deviceCategory: string;
browserName: string; // Per workflow-12102025.md: for session logging
// Derived headers for anti-detect
acceptLanguage: string;
secChUa?: string;
secChUaPlatform?: string;
secChUaMobile?: string;
// Per workflow-12102025.md: HTTP Fingerprinting section
httpFingerprint: HTTPFingerprint;
}
/**
* Per workflow-12102025.md: Session log entry for debugging blocked sessions
*/
export interface UASessionLog {
deviceCategory: string;
browserName: string;
userAgent: string;
proxyIp: string | null;
sessionStartedAt: Date;
}
// ============================================================
// PROXY ROTATOR CLASS
// ============================================================
@@ -91,9 +134,6 @@ export class ProxyRotator {
this.pool = pool || null;
}
/**
* Initialize with database pool
*/
setPool(pool: Pool): void {
this.pool = pool;
}
@@ -122,6 +162,7 @@ export class ProxyRotator {
0 as "successCount",
response_time_ms as "avgResponseTimeMs",
COALESCE(max_connections, 1) as "maxConnections",
COALESCE(consecutive_403_count, 0) as "consecutive403Count",
city,
state,
country,
@@ -134,11 +175,9 @@ export class ProxyRotator {
this.proxies = result.rows;
// Calculate total concurrent capacity
const totalCapacity = this.proxies.reduce((sum, p) => sum + p.maxConnections, 0);
console.log(`[ProxyRotator] Loaded ${this.proxies.length} active proxies (${totalCapacity} max concurrent connections)`);
} catch (error) {
// Table might not exist - that's okay
console.warn(`[ProxyRotator] Could not load proxies: ${error}`);
this.proxies = [];
}
@@ -150,7 +189,6 @@ export class ProxyRotator {
getNext(): Proxy | null {
if (this.proxies.length === 0) return null;
// Round-robin rotation
this.currentIndex = (this.currentIndex + 1) % this.proxies.length;
this.lastRotation = new Date();
@@ -185,23 +223,68 @@ export class ProxyRotator {
}
/**
* Mark proxy as failed (temporarily remove from rotation)
* Mark proxy as blocked (403 received)
* Per workflow-12102025.md:
* - Increment consecutive_403_count
* - After 3 consecutive 403s with different fingerprints → disable proxy
* - This is separate from general failures (timeouts, etc.)
*/
async markFailed(proxyId: number, error?: string): Promise<void> {
// Update in-memory
async markBlocked(proxyId: number): Promise<boolean> {
const proxy = this.proxies.find(p => p.id === proxyId);
if (proxy) {
proxy.failureCount++;
let shouldDisable = false;
// Deactivate if too many failures
if (proxy.failureCount >= 5) {
if (proxy) {
proxy.consecutive403Count++;
// Per workflow-12102025.md: 3 consecutive 403s → proxy is burned
if (proxy.consecutive403Count >= 3) {
proxy.isActive = false;
this.proxies = this.proxies.filter(p => p.id !== proxyId);
console.log(`[ProxyRotator] Proxy ${proxyId} deactivated after ${proxy.failureCount} failures`);
console.log(`[ProxyRotator] Proxy ${proxyId} DISABLED after ${proxy.consecutive403Count} consecutive 403s (burned)`);
shouldDisable = true;
} else {
console.log(`[ProxyRotator] Proxy ${proxyId} blocked (403 #${proxy.consecutive403Count}/3)`);
}
}
// Update database
if (this.pool) {
try {
await this.pool.query(`
UPDATE proxies
SET
consecutive_403_count = COALESCE(consecutive_403_count, 0) + 1,
last_failure_at = NOW(),
test_result = '403 Forbidden',
active = CASE WHEN COALESCE(consecutive_403_count, 0) >= 2 THEN false ELSE active END,
updated_at = NOW()
WHERE id = $1
`, [proxyId]);
} catch (err) {
console.error(`[ProxyRotator] Failed to update proxy ${proxyId}:`, err);
}
}
return shouldDisable;
}
/**
* Mark proxy as failed (general error - timeout, connection error, etc.)
* Separate from 403 blocking per workflow-12102025.md
*/
async markFailed(proxyId: number, error?: string): Promise<void> {
const proxy = this.proxies.find(p => p.id === proxyId);
if (proxy) {
proxy.failureCount++;
// Deactivate if too many general failures
if (proxy.failureCount >= 5) {
proxy.isActive = false;
this.proxies = this.proxies.filter(p => p.id !== proxyId);
console.log(`[ProxyRotator] Proxy ${proxyId} deactivated after ${proxy.failureCount} general failures`);
}
}
if (this.pool) {
try {
await this.pool.query(`
@@ -220,23 +303,22 @@ export class ProxyRotator {
}
/**
* Mark proxy as successful
* Mark proxy as successful - resets consecutive 403 count
* Per workflow-12102025.md: successful request clears the 403 counter
*/
async markSuccess(proxyId: number, responseTimeMs?: number): Promise<void> {
// Update in-memory
const proxy = this.proxies.find(p => p.id === proxyId);
if (proxy) {
proxy.successCount++;
proxy.consecutive403Count = 0; // Reset on success per workflow-12102025.md
proxy.lastUsedAt = new Date();
if (responseTimeMs !== undefined) {
// Rolling average
proxy.avgResponseTimeMs = proxy.avgResponseTimeMs
? (proxy.avgResponseTimeMs * 0.8) + (responseTimeMs * 0.2)
: responseTimeMs;
}
}
// Update database
if (this.pool) {
try {
await this.pool.query(`
@@ -244,6 +326,7 @@ export class ProxyRotator {
SET
last_tested_at = NOW(),
test_result = 'success',
consecutive_403_count = 0,
response_time_ms = CASE
WHEN response_time_ms IS NULL THEN $2
ELSE (response_time_ms * 0.8 + $2 * 0.2)::integer
@@ -272,8 +355,8 @@ export class ProxyRotator {
*/
getStats(): ProxyStats {
const totalProxies = this.proxies.length;
const activeProxies = this.proxies.reduce((sum, p) => sum + p.maxConnections, 0); // Total concurrent capacity
const blockedProxies = this.proxies.filter(p => p.failureCount >= 5).length;
const activeProxies = this.proxies.reduce((sum, p) => sum + p.maxConnections, 0);
const blockedProxies = this.proxies.filter(p => p.failureCount >= 5 || p.consecutive403Count >= 3).length;
const successRates = this.proxies
.filter(p => p.successCount + p.failureCount > 0)
@@ -285,15 +368,12 @@ export class ProxyRotator {
return {
totalProxies,
activeProxies, // Total concurrent capacity across all proxies
activeProxies,
blockedProxies,
avgSuccessRate,
};
}
/**
* Check if proxy pool has available proxies
*/
hasAvailableProxies(): boolean {
return this.proxies.length > 0;
}
@@ -301,53 +381,194 @@ export class ProxyRotator {
// ============================================================
// USER AGENT ROTATOR CLASS
// Per workflow-12102025.md: Uses intoli/user-agents for realistic fingerprints
// ============================================================
export class UserAgentRotator {
private userAgents: string[];
private currentIndex: number = 0;
private lastRotation: Date = new Date();
private currentFingerprint: BrowserFingerprint | null = null;
private sessionLog: UASessionLog | null = null;
constructor(userAgents: string[] = USER_AGENTS) {
this.userAgents = userAgents;
// Start at random index to avoid patterns
this.currentIndex = Math.floor(Math.random() * userAgents.length);
constructor() {
// Per workflow-12102025.md: Initialize with first fingerprint
this.rotate();
}
/**
* Get next user agent in rotation
* Per workflow-12102025.md: Roll device category based on distribution
* Mobile: 62%, Desktop: 36%, Tablet: 2%
*/
getNext(): string {
this.currentIndex = (this.currentIndex + 1) % this.userAgents.length;
this.lastRotation = new Date();
return this.userAgents[this.currentIndex];
private rollDeviceCategory(): 'mobile' | 'desktop' | 'tablet' {
const roll = Math.random() * 100;
if (roll < DEVICE_WEIGHTS.mobile) {
return 'mobile';
} else if (roll < DEVICE_WEIGHTS.mobile + DEVICE_WEIGHTS.desktop) {
return 'desktop';
} else {
return 'tablet';
}
}
/**
* Get current user agent without rotating
* Per workflow-12102025.md: Extract browser name from UA string
*/
getCurrent(): string {
return this.userAgents[this.currentIndex];
private extractBrowserName(userAgent: string): string {
if (userAgent.includes('Edg/')) return 'Edge';
if (userAgent.includes('Firefox/')) return 'Firefox';
if (userAgent.includes('Safari/') && !userAgent.includes('Chrome/')) return 'Safari';
if (userAgent.includes('Chrome/')) return 'Chrome';
return 'Unknown';
}
/**
* Get a random user agent
* Per workflow-12102025.md: Check if browser is in whitelist
*/
getRandom(): string {
const index = Math.floor(Math.random() * this.userAgents.length);
return this.userAgents[index];
private isAllowedBrowser(userAgent: string): boolean {
const browserName = this.extractBrowserName(userAgent);
return ALLOWED_BROWSERS.includes(browserName as typeof ALLOWED_BROWSERS[number]);
}
/**
* Get total available user agents
* Generate a new random fingerprint
* Per workflow-12102025.md:
* - Roll device category (62/36/2)
* - Filter to top 4 browsers only
* - Failure = alert admin + stop (no fallback)
*/
rotate(proxyIp?: string): BrowserFingerprint {
// Per workflow-12102025.md: Roll device category
const deviceCategory = this.rollDeviceCategory();
// Per workflow-12102025.md: Generate UA filtered to device category
const generator = new UserAgent({ deviceCategory });
// Per workflow-12102025.md: Try to get an allowed browser (max 50 attempts)
let ua: ReturnType<typeof generator>;
let attempts = 0;
const maxAttempts = 50;
do {
ua = generator();
attempts++;
} while (!this.isAllowedBrowser(ua.data.userAgent) && attempts < maxAttempts);
// Per workflow-12102025.md: If we can't get allowed browser, this is a failure
if (!this.isAllowedBrowser(ua.data.userAgent)) {
const errorMsg = `[UserAgentRotator] CRITICAL: Failed to generate allowed browser after ${maxAttempts} attempts. Device: ${deviceCategory}. Last UA: ${ua.data.userAgent}`;
console.error(errorMsg);
// Per workflow-12102025.md: Alert admin + stop crawl
// TODO: Post alert to admin dashboard
throw new Error(errorMsg);
}
const data = ua.data;
const browserName = this.extractBrowserName(data.userAgent);
// Build sec-ch-ua headers from user agent string
const secChUa = this.buildSecChUa(data.userAgent, deviceCategory);
// Per workflow-12102025.md: HTTP Fingerprinting - generate full HTTP fingerprint
const httpFingerprint = generateHTTPFingerprint(browserName as BrowserType);
this.currentFingerprint = {
userAgent: data.userAgent,
platform: data.platform,
screenWidth: data.screenWidth,
screenHeight: data.screenHeight,
viewportWidth: data.viewportWidth,
viewportHeight: data.viewportHeight,
deviceCategory: data.deviceCategory,
browserName, // Per workflow-12102025.md: for session logging
// Per workflow-12102025.md: always English
acceptLanguage: 'en-US,en;q=0.9',
...secChUa,
// Per workflow-12102025.md: HTTP Fingerprinting section
httpFingerprint,
};
// Per workflow-12102025.md: Log session data
this.sessionLog = {
deviceCategory,
browserName,
userAgent: data.userAgent,
proxyIp: proxyIp || null,
sessionStartedAt: new Date(),
};
console.log(`[UserAgentRotator] New fingerprint: device=${deviceCategory}, browser=${browserName}, UA=${data.userAgent.slice(0, 50)}...`);
return this.currentFingerprint;
}
/**
* Get current fingerprint without rotating
*/
getCurrent(): BrowserFingerprint {
if (!this.currentFingerprint) {
return this.rotate();
}
return this.currentFingerprint;
}
/**
* Get a random fingerprint (rotates and returns)
*/
getRandom(proxyIp?: string): BrowserFingerprint {
return this.rotate(proxyIp);
}
/**
* Per workflow-12102025.md: Get session log for debugging
*/
getSessionLog(): UASessionLog | null {
return this.sessionLog;
}
/**
* Build sec-ch-ua headers from user agent string
* Per workflow-12102025.md: Include mobile indicator based on device category
*/
private buildSecChUa(userAgent: string, deviceCategory: string): { secChUa?: string; secChUaPlatform?: string; secChUaMobile?: string } {
const isMobile = deviceCategory === 'mobile' || deviceCategory === 'tablet';
// Extract Chrome version if present
const chromeMatch = userAgent.match(/Chrome\/(\d+)/);
const edgeMatch = userAgent.match(/Edg\/(\d+)/);
if (edgeMatch) {
const version = edgeMatch[1];
return {
secChUa: `"Microsoft Edge";v="${version}", "Chromium";v="${version}", "Not_A Brand";v="24"`,
secChUaPlatform: userAgent.includes('Windows') ? '"Windows"' : userAgent.includes('Android') ? '"Android"' : '"macOS"',
secChUaMobile: isMobile ? '?1' : '?0',
};
}
if (chromeMatch) {
const version = chromeMatch[1];
let platform = '"Linux"';
if (userAgent.includes('Windows')) platform = '"Windows"';
else if (userAgent.includes('Mac')) platform = '"macOS"';
else if (userAgent.includes('Android')) platform = '"Android"';
else if (userAgent.includes('iPhone') || userAgent.includes('iPad')) platform = '"iOS"';
return {
secChUa: `"Google Chrome";v="${version}", "Chromium";v="${version}", "Not_A Brand";v="24"`,
secChUaPlatform: platform,
secChUaMobile: isMobile ? '?1' : '?0',
};
}
// Firefox/Safari don't send sec-ch-ua
return {};
}
getCount(): number {
return this.userAgents.length;
return 1; // user-agents generates dynamically
}
}
// ============================================================
// COMBINED ROTATOR (for convenience)
// COMBINED ROTATOR
// Per workflow-12102025.md: Coordinates proxy + fingerprint rotation
// ============================================================
export class CrawlRotator {
@@ -359,49 +580,51 @@ export class CrawlRotator {
this.userAgent = new UserAgentRotator();
}
/**
* Initialize rotator (load proxies from DB)
*/
async initialize(): Promise<void> {
await this.proxy.loadProxies();
}
/**
* Rotate proxy only
* Rotate proxy only (get new IP)
*/
rotateProxy(): Proxy | null {
return this.proxy.getNext();
}
/**
* Rotate user agent only
* Rotate fingerprint only (new UA, screen size, etc.)
*/
rotateUserAgent(): string {
return this.userAgent.getNext();
rotateFingerprint(): BrowserFingerprint {
return this.userAgent.rotate();
}
/**
* Rotate both proxy and user agent
* Rotate both proxy and fingerprint
* Per workflow-12102025.md: called on 403 for fresh identity
* Passes proxy IP to UA rotation for session logging
*/
rotateBoth(): { proxy: Proxy | null; userAgent: string } {
rotateBoth(): { proxy: Proxy | null; fingerprint: BrowserFingerprint } {
const proxy = this.proxy.getNext();
const proxyIp = proxy ? proxy.host : undefined;
return {
proxy: this.proxy.getNext(),
userAgent: this.userAgent.getNext(),
proxy,
fingerprint: this.userAgent.rotate(proxyIp),
};
}
/**
* Get current proxy and user agent without rotating
* Get current proxy and fingerprint without rotating
*/
getCurrent(): { proxy: Proxy | null; userAgent: string } {
getCurrent(): { proxy: Proxy | null; fingerprint: BrowserFingerprint } {
return {
proxy: this.proxy.getCurrent(),
userAgent: this.userAgent.getCurrent(),
fingerprint: this.userAgent.getCurrent(),
};
}
/**
* Record success for current proxy
* Per workflow-12102025.md: resets consecutive 403 count
*/
async recordSuccess(responseTimeMs?: number): Promise<void> {
const current = this.proxy.getCurrent();
@@ -411,7 +634,20 @@ export class CrawlRotator {
}
/**
* Record failure for current proxy
* Record 403 block for current proxy
* Per workflow-12102025.md: increments consecutive_403_count, disables after 3
* Returns true if proxy was disabled
*/
async recordBlock(): Promise<boolean> {
const current = this.proxy.getCurrent();
if (current) {
return await this.proxy.markBlocked(current.id);
}
return false;
}
/**
* Record general failure (not 403)
*/
async recordFailure(error?: string): Promise<void> {
const current = this.proxy.getCurrent();
@@ -421,14 +657,13 @@ export class CrawlRotator {
}
/**
* Get current proxy location info (for reporting)
* Note: For rotating proxies (like IPRoyal), the actual exit location varies per request
* Get current proxy location info
* Per workflow-12102025.md: proxy location determines session headers
*/
getProxyLocation(): { city?: string; state?: string; country?: string; timezone?: string; isRotating: boolean } | null {
const current = this.proxy.getCurrent();
if (!current) return null;
// Check if this is a rotating proxy (max_connections > 1 usually indicates rotating)
const isRotating = current.maxConnections > 1;
return {
@@ -439,6 +674,15 @@ export class CrawlRotator {
isRotating
};
}
/**
* Get timezone from current proxy
* Per workflow-12102025.md: used for Accept-Language header
*/
getProxyTimezone(): string | undefined {
const current = this.proxy.getCurrent();
return current?.timezone;
}
}
// ============================================================

315
backend/src/services/http-fingerprint.ts Normal file
View File

@@ -0,0 +1,315 @@
/**
* HTTP Fingerprinting Service
*
* Per workflow-12102025.md - HTTP Fingerprinting section:
* - Full header set per browser type
* - Browser-specific header ordering
* - Natural randomization (DNT, Accept quality)
* - Dynamic Referer per dispensary
*
* Canonical location: src/services/http-fingerprint.ts
*/
// ============================================================
// TYPES
// ============================================================
export type BrowserType = 'Chrome' | 'Firefox' | 'Safari' | 'Edge';
/**
* Per workflow-12102025.md: Full HTTP fingerprint for a session
*/
export interface HTTPFingerprint {
browserType: BrowserType;
headers: Record<string, string>;
headerOrder: string[];
curlImpersonateBinary: string;
hasDNT: boolean;
}
/**
* Per workflow-12102025.md: Context for building headers
*/
export interface HeaderContext {
userAgent: string;
secChUa?: string;
secChUaPlatform?: string;
secChUaMobile?: string;
referer: string;
isPost: boolean;
contentLength?: number;
}
// ============================================================
// CONSTANTS (per workflow-12102025.md)
// ============================================================
/**
* Per workflow-12102025.md: DNT header distribution (~30% of users)
*/
const DNT_PROBABILITY = 0.30;
/**
* Per workflow-12102025.md: Accept header variations for natural traffic
*/
const ACCEPT_VARIATIONS = [
'application/json, text/plain, */*',
'application/json,text/plain,*/*',
'*/*',
];
/**
* Per workflow-12102025.md: Accept-Language variations
*/
const ACCEPT_LANGUAGE_VARIATIONS = [
'en-US,en;q=0.9',
'en-US,en;q=0.8',
'en-US;q=0.9,en;q=0.8',
];
/**
* Per workflow-12102025.md: curl-impersonate binaries per browser
*/
const CURL_IMPERSONATE_BINARIES: Record<BrowserType, string> = {
Chrome: 'curl_chrome131',
Edge: 'curl_chrome131', // Edge uses Chromium
Firefox: 'curl_ff133',
Safari: 'curl_safari17',
};
// ============================================================
// HEADER ORDERING (per workflow-12102025.md)
// ============================================================
/**
* Per workflow-12102025.md: Chrome header order for GraphQL requests
*/
const CHROME_HEADER_ORDER = [
'Host',
'Connection',
'Content-Length',
'sec-ch-ua',
'DNT',
'sec-ch-ua-mobile',
'User-Agent',
'sec-ch-ua-platform',
'Content-Type',
'Accept',
'Origin',
'sec-fetch-site',
'sec-fetch-mode',
'sec-fetch-dest',
'Referer',
'Accept-Encoding',
'Accept-Language',
];
/**
* Per workflow-12102025.md: Firefox header order for GraphQL requests
*/
const FIREFOX_HEADER_ORDER = [
'Host',
'User-Agent',
'Accept',
'Accept-Language',
'Accept-Encoding',
'Content-Type',
'Content-Length',
'Origin',
'DNT',
'Connection',
'Referer',
'sec-fetch-dest',
'sec-fetch-mode',
'sec-fetch-site',
];
/**
* Per workflow-12102025.md: Safari header order for GraphQL requests
*/
const SAFARI_HEADER_ORDER = [
'Host',
'Connection',
'Content-Length',
'Accept',
'User-Agent',
'Content-Type',
'Origin',
'Referer',
'Accept-Encoding',
'Accept-Language',
];
/**
* Per workflow-12102025.md: Edge uses Chrome order (Chromium-based)
*/
const HEADER_ORDERS: Record<BrowserType, string[]> = {
Chrome: CHROME_HEADER_ORDER,
Edge: CHROME_HEADER_ORDER,
Firefox: FIREFOX_HEADER_ORDER,
Safari: SAFARI_HEADER_ORDER,
};
// ============================================================
// FINGERPRINT GENERATION
// ============================================================
/**
* Per workflow-12102025.md: Generate HTTP fingerprint for a session
* Randomization is done once per session for consistency
*/
export function generateHTTPFingerprint(browserType: BrowserType): HTTPFingerprint {
// Per workflow-12102025.md: DNT randomized per session (~30%)
const hasDNT = Math.random() < DNT_PROBABILITY;
return {
browserType,
headers: {}, // Built dynamically per request
headerOrder: HEADER_ORDERS[browserType],
curlImpersonateBinary: CURL_IMPERSONATE_BINARIES[browserType],
hasDNT,
};
}
/**
* Per workflow-12102025.md: Build complete headers for a request
* Returns headers in browser-specific order
*/
export function buildOrderedHeaders(
fingerprint: HTTPFingerprint,
context: HeaderContext
): { headers: Record<string, string>; orderedHeaders: string[] } {
const { browserType, hasDNT, headerOrder } = fingerprint;
const { userAgent, secChUa, secChUaPlatform, secChUaMobile, referer, isPost, contentLength } = context;
// Per workflow-12102025.md: Natural randomization for Accept
const accept = ACCEPT_VARIATIONS[Math.floor(Math.random() * ACCEPT_VARIATIONS.length)];
const acceptLanguage = ACCEPT_LANGUAGE_VARIATIONS[Math.floor(Math.random() * ACCEPT_LANGUAGE_VARIATIONS.length)];
// Build all possible headers
const allHeaders: Record<string, string> = {
'Connection': 'keep-alive',
'User-Agent': userAgent,
'Accept': accept,
'Accept-Language': acceptLanguage,
'Accept-Encoding': 'gzip, deflate, br',
};
// Per workflow-12102025.md: POST-only headers
if (isPost) {
allHeaders['Content-Type'] = 'application/json';
allHeaders['Origin'] = 'https://dutchie.com';
if (contentLength !== undefined) {
allHeaders['Content-Length'] = String(contentLength);
}
}
// Per workflow-12102025.md: Dynamic Referer per dispensary
allHeaders['Referer'] = referer;
// Per workflow-12102025.md: DNT randomized per session
if (hasDNT) {
allHeaders['DNT'] = '1';
}
// Per workflow-12102025.md: Chromium-only headers (Chrome, Edge)
if (browserType === 'Chrome' || browserType === 'Edge') {
if (secChUa) allHeaders['sec-ch-ua'] = secChUa;
if (secChUaMobile) allHeaders['sec-ch-ua-mobile'] = secChUaMobile;
if (secChUaPlatform) allHeaders['sec-ch-ua-platform'] = secChUaPlatform;
allHeaders['sec-fetch-site'] = 'same-origin';
allHeaders['sec-fetch-mode'] = 'cors';
allHeaders['sec-fetch-dest'] = 'empty';
}
// Per workflow-12102025.md: Firefox has sec-fetch but no sec-ch
if (browserType === 'Firefox') {
allHeaders['sec-fetch-site'] = 'same-origin';
allHeaders['sec-fetch-mode'] = 'cors';
allHeaders['sec-fetch-dest'] = 'empty';
}
// Per workflow-12102025.md: Safari has no sec-* headers
// Filter to only headers that exist and order them
const orderedHeaders: string[] = [];
const headers: Record<string, string> = {};
for (const headerName of headerOrder) {
if (allHeaders[headerName]) {
orderedHeaders.push(headerName);
headers[headerName] = allHeaders[headerName];
}
}
return { headers, orderedHeaders };
}
/**
* Per workflow-12102025.md: Build curl command arguments for headers
* Headers are added in browser-specific order
*/
export function buildCurlHeaderArgs(
fingerprint: HTTPFingerprint,
context: HeaderContext
): string[] {
const { headers, orderedHeaders } = buildOrderedHeaders(fingerprint, context);
const args: string[] = [];
for (const headerName of orderedHeaders) {
// Skip Host and Content-Length - curl handles these
if (headerName === 'Host' || headerName === 'Content-Length') continue;
args.push('-H', `${headerName}: ${headers[headerName]}`);
}
return args;
}
/**
* Per workflow-12102025.md: Extract Referer from dispensary menu_url
*/
export function buildRefererFromMenuUrl(menuUrl: string | null | undefined): string {
if (!menuUrl) {
return 'https://dutchie.com/';
}
// Extract slug from menu_url
// Formats: /embedded-menu/<slug> or /dispensary/<slug> or full URL
let slug: string | null = null;
const embeddedMatch = menuUrl.match(/\/embedded-menu\/([^/?]+)/);
const dispensaryMatch = menuUrl.match(/\/dispensary\/([^/?]+)/);
if (embeddedMatch) {
slug = embeddedMatch[1];
} else if (dispensaryMatch) {
slug = dispensaryMatch[1];
}
if (slug) {
return `https://dutchie.com/dispensary/${slug}`;
}
return 'https://dutchie.com/';
}
/**
* Per workflow-12102025.md: Get curl-impersonate binary for browser
*/
export function getCurlBinary(browserType: BrowserType): string {
return CURL_IMPERSONATE_BINARIES[browserType];
}
/**
* Per workflow-12102025.md: Check if curl-impersonate is available
*/
export function isCurlImpersonateAvailable(browserType: BrowserType): boolean {
const binary = CURL_IMPERSONATE_BINARIES[browserType];
try {
const { execSync } = require('child_process');
execSync(`which ${binary}`, { stdio: 'ignore' });
return true;
} catch {
return false;
}
}

126
backend/src/services/scheduler.ts
View File

@@ -1,116 +1,38 @@
import cron from 'node-cron';
import { pool } from '../db/pool';
import { scrapeStore, scrapeCategory } from '../scraper-v2';
let scheduledJobs: cron.ScheduledTask[] = [];
async function getSettings(): Promise<{
scrapeIntervalHours: number;
scrapeSpecialsTime: string;
}> {
const result = await pool.query(`
SELECT key, value FROM settings
WHERE key IN ('scrape_interval_hours', 'scrape_specials_time')
`);
const settings: Record<string, string> = {};
result.rows.forEach((row: { key: string; value: string }) => {
settings[row.key] = row.value;
});
return {
scrapeIntervalHours: parseInt(settings.scrape_interval_hours || '4'),
scrapeSpecialsTime: settings.scrape_specials_time || '00:01'
};
}
async function scrapeAllStores(): Promise<void> {
console.log('🔄 Starting scheduled scrape for all stores...');
const result = await pool.query(`
SELECT id, name FROM stores WHERE active = true AND scrape_enabled = true
`);
for (const store of result.rows) {
try {
console.log(`Scraping store: ${store.name}`);
await scrapeStore(store.id);
} catch (error) {
console.error(`Failed to scrape store ${store.name}:`, error);
}
}
console.log('✅ Scheduled scrape completed');
}
async function scrapeSpecials(): Promise<void> {
console.log('🌟 Starting scheduled specials scrape...');
const result = await pool.query(`
SELECT s.id, s.name, c.id as category_id
FROM stores s
JOIN categories c ON c.store_id = s.id
WHERE s.active = true AND s.scrape_enabled = true
AND c.slug = 'specials' AND c.scrape_enabled = true
`);
for (const row of result.rows) {
try {
console.log(`Scraping specials for: ${row.name}`);
await scrapeCategory(row.id, row.category_id);
} catch (error) {
console.error(`Failed to scrape specials for ${row.name}:`, error);
}
}
console.log('✅ Specials scrape completed');
}
/**
* LEGACY SCHEDULER - DEPRECATED 2024-12-10
*
* DO NOT USE THIS FILE.
*
* Per TASK_WORKFLOW_2024-12-10.md:
* This node-cron scheduler has been replaced by the database-driven
* task scheduler in src/services/task-scheduler.ts
*
* The new scheduler:
* - Stores schedules in PostgreSQL (survives restarts)
* - Uses SELECT FOR UPDATE SKIP LOCKED (multi-replica safe)
* - Creates tasks in worker_tasks table (processed by task-worker.ts)
*
* This file is kept for reference only. All exports are no-ops.
* Legacy code has been removed - see git history for original implementation.
*/
// 2024-12-10: All functions are now no-ops
export async function startScheduler(): Promise<void> {
// Stop any existing jobs
stopScheduler();
const settings = await getSettings();
// Schedule regular store scrapes (every N hours)
const scrapeIntervalCron = `0 */${settings.scrapeIntervalHours} * * *`;
const storeJob = cron.schedule(scrapeIntervalCron, scrapeAllStores);
scheduledJobs.push(storeJob);
console.log(`📅 Scheduled store scraping: every ${settings.scrapeIntervalHours} hours`);
// Schedule specials scraping (daily at specified time)
const [hours, minutes] = settings.scrapeSpecialsTime.split(':');
const specialsCron = `${minutes} ${hours} * * *`;
const specialsJob = cron.schedule(specialsCron, scrapeSpecials);
scheduledJobs.push(specialsJob);
console.log(`📅 Scheduled specials scraping: daily at ${settings.scrapeSpecialsTime}`);
// Initial scrape on startup (after 10 seconds)
setTimeout(() => {
console.log('🚀 Running initial scrape...');
scrapeAllStores().catch(console.error);
}, 10000);
console.warn('[DEPRECATED] startScheduler() called - use taskScheduler from task-scheduler.ts instead');
}
export function stopScheduler(): void {
scheduledJobs.forEach(job => job.stop());
scheduledJobs = [];
console.log('🛑 Scheduler stopped');
console.warn('[DEPRECATED] stopScheduler() called - use taskScheduler from task-scheduler.ts instead');
}
export async function restartScheduler(): Promise<void> {
console.log('🔄 Restarting scheduler...');
stopScheduler();
await startScheduler();
console.warn('[DEPRECATED] restartScheduler() called - use taskScheduler from task-scheduler.ts instead');
}
// Manual trigger functions for admin
export async function triggerStoreScrape(storeId: number): Promise<void> {
console.log(`🔧 Manual scrape triggered for store ID: ${storeId}`);
await scrapeStore(storeId);
export async function triggerStoreScrape(_storeId: number): Promise<void> {
console.warn('[DEPRECATED] triggerStoreScrape() called - use taskService.createTask() instead');
}
export async function triggerAllStoresScrape(): Promise<void> {
console.log('🔧 Manual scrape triggered for all stores');
await scrapeAllStores();
console.warn('[DEPRECATED] triggerAllStoresScrape() called - use taskScheduler.triggerSchedule() instead');
}

375
backend/src/services/task-scheduler.ts Normal file
View File

@@ -0,0 +1,375 @@
/**
* Database-Driven Task Scheduler
*
* Per TASK_WORKFLOW_2024-12-10.md:
* - Schedules stored in DB (survives restarts)
* - Uses SELECT FOR UPDATE to prevent duplicate execution across replicas
* - Polls every 60s to check if schedules are due
* - Generates tasks into worker_tasks table for task-worker.ts to process
*
* 2024-12-10: Created to replace legacy node-cron scheduler
*/
import { pool } from '../db/pool';
import { taskService, TaskRole } from '../tasks/task-service';
// Per TASK_WORKFLOW_2024-12-10.md: Poll interval for checking schedules
const POLL_INTERVAL_MS = 60_000; // 60 seconds
interface TaskSchedule {
id: number;
name: string;
role: TaskRole;
enabled: boolean;
interval_hours: number;
last_run_at: Date | null;
next_run_at: Date | null;
state_code: string | null;
priority: number;
}
class TaskScheduler {
private pollTimer: NodeJS.Timeout | null = null;
private isRunning = false;
/**
* Start the scheduler
* Per TASK_WORKFLOW_2024-12-10.md: Called on API server startup
*/
async start(): Promise<void> {
if (this.isRunning) {
console.log('[TaskScheduler] Already running');
return;
}
console.log('[TaskScheduler] Starting database-driven scheduler...');
this.isRunning = true;
// Per TASK_WORKFLOW_2024-12-10.md: On startup, recover stale tasks
try {
const recovered = await taskService.recoverStaleTasks(10);
if (recovered > 0) {
console.log(`[TaskScheduler] Recovered ${recovered} stale tasks from dead workers`);
}
} catch (err: any) {
console.error('[TaskScheduler] Failed to recover stale tasks:', err.message);
}
// Per TASK_WORKFLOW_2024-12-10.md: Ensure default schedules exist
await this.ensureDefaultSchedules();
// Per TASK_WORKFLOW_2024-12-10.md: Check immediately on startup
await this.checkAndRunDueSchedules();
// Per TASK_WORKFLOW_2024-12-10.md: Then poll every 60 seconds
this.pollTimer = setInterval(async () => {
await this.checkAndRunDueSchedules();
}, POLL_INTERVAL_MS);
console.log('[TaskScheduler] Started - polling every 60s');
}
/**
* Stop the scheduler
*/
stop(): void {
if (this.pollTimer) {
clearInterval(this.pollTimer);
this.pollTimer = null;
}
this.isRunning = false;
console.log('[TaskScheduler] Stopped');
}
/**
* Ensure default schedules exist in the database
* Per TASK_WORKFLOW_2024-12-10.md: Creates schedules if they don't exist
*/
private async ensureDefaultSchedules(): Promise<void> {
// Per TASK_WORKFLOW_2024-12-10.md: Default schedules for task generation
// NOTE: payload_fetch replaces direct product_refresh - it chains to product_refresh
const defaults = [
{
name: 'payload_fetch_all',
role: 'payload_fetch' as TaskRole,
interval_hours: 4,
priority: 0,
description: 'Fetch payloads from Dutchie API for all crawl-enabled stores every 4 hours. Chains to product_refresh.',
},
{
name: 'store_discovery_dutchie',
role: 'store_discovery' as TaskRole,
interval_hours: 24,
priority: 5,
description: 'Discover new Dutchie stores daily',
},
{
name: 'analytics_refresh',
role: 'analytics_refresh' as TaskRole,
interval_hours: 6,
priority: 0,
description: 'Refresh analytics materialized views every 6 hours',
},
];
for (const sched of defaults) {
try {
await pool.query(`
INSERT INTO task_schedules (name, role, interval_hours, priority, description, enabled, next_run_at)
VALUES ($1, $2, $3, $4, $5, true, NOW())
ON CONFLICT (name) DO NOTHING
`, [sched.name, sched.role, sched.interval_hours, sched.priority, sched.description]);
} catch (err: any) {
// Table may not exist yet - will be created by migration
if (!err.message.includes('does not exist')) {
console.error(`[TaskScheduler] Failed to create default schedule ${sched.name}:`, err.message);
}
}
}
}
/**
* Check for and run any due schedules
* Per TASK_WORKFLOW_2024-12-10.md: Uses SELECT FOR UPDATE SKIP LOCKED to prevent duplicates
*/
private async checkAndRunDueSchedules(): Promise<void> {
const client = await pool.connect();
try {
await client.query('BEGIN');
// Per TASK_WORKFLOW_2024-12-10.md: Atomic claim of due schedules
const result = await client.query<TaskSchedule>(`
SELECT *
FROM task_schedules
WHERE enabled = true
AND (next_run_at IS NULL OR next_run_at <= NOW())
FOR UPDATE SKIP LOCKED
`);
for (const schedule of result.rows) {
console.log(`[TaskScheduler] Running schedule: ${schedule.name} (${schedule.role})`);
try {
const tasksCreated = await this.executeSchedule(schedule);
console.log(`[TaskScheduler] Schedule ${schedule.name} created ${tasksCreated} tasks`);
// Per TASK_WORKFLOW_2024-12-10.md: Update last_run_at and calculate next_run_at
await client.query(`
UPDATE task_schedules
SET
last_run_at = NOW(),
next_run_at = NOW() + ($1 || ' hours')::interval,
last_task_count = $2,
updated_at = NOW()
WHERE id = $3
`, [schedule.interval_hours, tasksCreated, schedule.id]);
} catch (err: any) {
console.error(`[TaskScheduler] Schedule ${schedule.name} failed:`, err.message);
// Still update next_run_at to prevent infinite retry loop
await client.query(`
UPDATE task_schedules
SET
next_run_at = NOW() + ($1 || ' hours')::interval,
last_error = $2,
updated_at = NOW()
WHERE id = $3
`, [schedule.interval_hours, err.message, schedule.id]);
}
}
await client.query('COMMIT');
} catch (err: any) {
await client.query('ROLLBACK');
console.error('[TaskScheduler] Failed to check schedules:', err.message);
} finally {
client.release();
}
}
/**
* Execute a schedule and create tasks
* Per TASK_WORKFLOW_2024-12-10.md: Different logic per role
*/
private async executeSchedule(schedule: TaskSchedule): Promise<number> {
switch (schedule.role) {
case 'payload_fetch':
// Per TASK_WORKFLOW_2024-12-10.md: payload_fetch replaces direct product_refresh
return this.generatePayloadFetchTasks(schedule);
case 'product_refresh':
// Legacy - kept for manual triggers, but scheduled crawls use payload_fetch
return this.generatePayloadFetchTasks(schedule);
case 'store_discovery':
return this.generateStoreDiscoveryTasks(schedule);
case 'analytics_refresh':
return this.generateAnalyticsRefreshTasks(schedule);
default:
console.warn(`[TaskScheduler] Unknown role: ${schedule.role}`);
return 0;
}
}
/**
* Generate payload_fetch tasks for stores that need crawling
* Per TASK_WORKFLOW_2024-12-10.md: payload_fetch hits API, saves to disk, chains to product_refresh
*/
private async generatePayloadFetchTasks(schedule: TaskSchedule): Promise<number> {
// Per TASK_WORKFLOW_2024-12-10.md: Find stores needing refresh
const result = await pool.query(`
SELECT d.id
FROM dispensaries d
WHERE d.crawl_enabled = true
AND d.platform_dispensary_id IS NOT NULL
-- No pending/running payload_fetch or product_refresh task already
AND NOT EXISTS (
SELECT 1 FROM worker_tasks t
WHERE t.dispensary_id = d.id
AND t.role IN ('payload_fetch', 'product_refresh')
AND t.status IN ('pending', 'claimed', 'running')
)
-- Never fetched OR last fetch > interval ago
AND (
d.last_fetch_at IS NULL
OR d.last_fetch_at < NOW() - ($1 || ' hours')::interval
)
${schedule.state_code ? 'AND d.state_id = (SELECT id FROM states WHERE code = $2)' : ''}
`, schedule.state_code ? [schedule.interval_hours, schedule.state_code] : [schedule.interval_hours]);
const dispensaryIds = result.rows.map((r: { id: number }) => r.id);
if (dispensaryIds.length === 0) {
return 0;
}
// Per TASK_WORKFLOW_2024-12-10.md: Create payload_fetch tasks (they chain to product_refresh)
const tasks = dispensaryIds.map((id: number) => ({
role: 'payload_fetch' as TaskRole,
dispensary_id: id,
priority: schedule.priority,
}));
return taskService.createTasks(tasks);
}
/**
* Generate store_discovery tasks
* Per TASK_WORKFLOW_2024-12-10.md: One task per platform
*/
private async generateStoreDiscoveryTasks(schedule: TaskSchedule): Promise<number> {
// Check if discovery task already pending
const existing = await taskService.listTasks({
role: 'store_discovery',
status: ['pending', 'claimed', 'running'],
limit: 1,
});
if (existing.length > 0) {
console.log('[TaskScheduler] Store discovery task already pending, skipping');
return 0;
}
await taskService.createTask({
role: 'store_discovery',
platform: 'dutchie',
priority: schedule.priority,
});
return 1;
}
/**
* Generate analytics_refresh tasks
* Per TASK_WORKFLOW_2024-12-10.md: Single task to refresh all MVs
*/
private async generateAnalyticsRefreshTasks(schedule: TaskSchedule): Promise<number> {
// Check if analytics task already pending
const existing = await taskService.listTasks({
role: 'analytics_refresh',
status: ['pending', 'claimed', 'running'],
limit: 1,
});
if (existing.length > 0) {
console.log('[TaskScheduler] Analytics refresh task already pending, skipping');
return 0;
}
await taskService.createTask({
role: 'analytics_refresh',
priority: schedule.priority,
});
return 1;
}
/**
* Get all schedules for dashboard display
*/
async getSchedules(): Promise<TaskSchedule[]> {
try {
const result = await pool.query(`
SELECT * FROM task_schedules ORDER BY name
`);
return result.rows as TaskSchedule[];
} catch {
return [];
}
}
/**
* Update a schedule
*/
async updateSchedule(id: number, updates: Partial<TaskSchedule>): Promise<void> {
const setClauses: string[] = [];
const values: any[] = [];
let paramIndex = 1;
if (updates.enabled !== undefined) {
setClauses.push(`enabled = $${paramIndex++}`);
values.push(updates.enabled);
}
if (updates.interval_hours !== undefined) {
setClauses.push(`interval_hours = $${paramIndex++}`);
values.push(updates.interval_hours);
}
if (updates.priority !== undefined) {
setClauses.push(`priority = $${paramIndex++}`);
values.push(updates.priority);
}
if (setClauses.length === 0) return;
setClauses.push('updated_at = NOW()');
values.push(id);
await pool.query(`
UPDATE task_schedules
SET ${setClauses.join(', ')}
WHERE id = $${paramIndex}
`, values);
}
/**
* Trigger a schedule to run immediately
*/
async triggerSchedule(id: number): Promise<number> {
const result = await pool.query(`
SELECT * FROM task_schedules WHERE id = $1
`, [id]);
if (result.rows.length === 0) {
throw new Error(`Schedule ${id} not found`);
}
return this.executeSchedule(result.rows[0] as TaskSchedule);
}
}
// Per TASK_WORKFLOW_2024-12-10.md: Singleton instance
export const taskScheduler = new TaskScheduler();

3
backend/src/tasks/handlers/entry-point-discovery.ts
View File

@@ -94,7 +94,8 @@ export async function handleEntryPointDiscovery(ctx: TaskContext): Promise<TaskR
// ============================================================
// STEP 3: Start stealth session
// ============================================================
const session = startSession(dispensary.state || 'AZ', 'America/Phoenix');
// Per workflow-12102025.md: session identity comes from proxy location, not task params
const session = startSession();
console.log(`[EntryPointDiscovery] Session started: ${session.sessionId}`);
try {

221
backend/src/tasks/handlers/payload-fetch.ts Normal file
View File

@@ -0,0 +1,221 @@
/**
* Payload Fetch Handler
*
* Per TASK_WORKFLOW_2024-12-10.md: Separates API fetch from data processing.
*
* This handler ONLY:
* 1. Hits Dutchie GraphQL API
* 2. Saves raw payload to filesystem (gzipped)
* 3. Records metadata in raw_crawl_payloads table
* 4. Queues a product_refresh task to process the payload
*
* Benefits of separation:
* - Retry-friendly: If normalize fails, re-run refresh without re-crawling
* - Faster refreshes: Local file read vs network call
* - Replay-able: Run refresh against any historical payload
* - Less API pressure: Only this role hits Dutchie
*/
import { TaskContext, TaskResult } from '../task-worker';
import {
executeGraphQL,
startSession,
endSession,
GRAPHQL_HASHES,
DUTCHIE_CONFIG,
} from '../../platforms/dutchie';
import { saveRawPayload } from '../../utils/payload-storage';
import { taskService } from '../task-service';
export async function handlePayloadFetch(ctx: TaskContext): Promise<TaskResult> {
const { pool, task } = ctx;
const dispensaryId = task.dispensary_id;
if (!dispensaryId) {
return { success: false, error: 'No dispensary_id specified for payload_fetch task' };
}
try {
// ============================================================
// STEP 1: Load dispensary info
// ============================================================
const dispResult = await pool.query(`
SELECT
id, name, platform_dispensary_id, menu_url, menu_type, city, state
FROM dispensaries
WHERE id = $1 AND crawl_enabled = true
`, [dispensaryId]);
if (dispResult.rows.length === 0) {
return { success: false, error: `Dispensary ${dispensaryId} not found or not crawl_enabled` };
}
const dispensary = dispResult.rows[0];
const platformId = dispensary.platform_dispensary_id;
if (!platformId) {
return { success: false, error: `Dispensary ${dispensaryId} has no platform_dispensary_id` };
}
// Extract cName from menu_url
const cNameMatch = dispensary.menu_url?.match(/\/(?:embedded-menu|dispensary)\/([^/?]+)/);
const cName = cNameMatch ? cNameMatch[1] : 'dispensary';
console.log(`[PayloadFetch] Starting fetch for ${dispensary.name} (ID: ${dispensaryId})`);
console.log(`[PayloadFetch] Platform ID: ${platformId}, cName: ${cName}`);
// ============================================================
// STEP 2: Start stealth session
// ============================================================
const session = startSession();
console.log(`[PayloadFetch] Session started: ${session.sessionId}`);
await ctx.heartbeat();
// ============================================================
// STEP 3: Fetch products via GraphQL (Status: 'All')
// ============================================================
const allProducts: any[] = [];
let page = 0;
let totalCount = 0;
const perPage = DUTCHIE_CONFIG.perPage;
const maxPages = DUTCHIE_CONFIG.maxPages;
try {
while (page < maxPages) {
const variables = {
includeEnterpriseSpecials: false,
productsFilter: {
dispensaryId: platformId,
pricingType: 'rec',
Status: 'All',
types: [],
useCache: false,
isDefaultSort: true,
sortBy: 'popularSortIdx',
sortDirection: 1,
bypassOnlineThresholds: true,
isKioskMenu: false,
removeProductsBelowOptionThresholds: false,
},
page,
perPage,
};
console.log(`[PayloadFetch] Fetching page ${page + 1}...`);
const result = await executeGraphQL(
'FilteredProducts',
variables,
GRAPHQL_HASHES.FilteredProducts,
{ cName, maxRetries: 3 }
);
const data = result?.data?.filteredProducts;
if (!data || !data.products) {
if (page === 0) {
throw new Error('No product data returned from GraphQL');
}
break;
}
const products = data.products;
allProducts.push(...products);
if (page === 0) {
totalCount = data.queryInfo?.totalCount || products.length;
console.log(`[PayloadFetch] Total products reported: ${totalCount}`);
}
if (allProducts.length >= totalCount || products.length < perPage) {
break;
}
page++;
if (page < maxPages) {
await new Promise(r => setTimeout(r, DUTCHIE_CONFIG.pageDelayMs));
}
if (page % 5 === 0) {
await ctx.heartbeat();
}
}
console.log(`[PayloadFetch] Fetched ${allProducts.length} products in ${page + 1} pages`);
} finally {
endSession();
}
if (allProducts.length === 0) {
return {
success: false,
error: 'No products returned from GraphQL',
productsProcessed: 0,
};
}
await ctx.heartbeat();
// ============================================================
// STEP 4: Save raw payload to filesystem
// Per TASK_WORKFLOW_2024-12-10.md: Metadata/Payload separation
// ============================================================
const rawPayload = {
dispensaryId,
platformId,
cName,
fetchedAt: new Date().toISOString(),
productCount: allProducts.length,
products: allProducts,
};
const payloadResult = await saveRawPayload(
pool,
dispensaryId,
rawPayload,
null, // crawl_run_id - not using crawl_runs in new system
allProducts.length
);
console.log(`[PayloadFetch] Saved payload #${payloadResult.id} (${(payloadResult.sizeBytes / 1024).toFixed(1)}KB)`);
// ============================================================
// STEP 5: Update dispensary last_fetch_at
// ============================================================
await pool.query(`
UPDATE dispensaries
SET last_fetch_at = NOW()
WHERE id = $1
`, [dispensaryId]);
// ============================================================
// STEP 6: Queue product_refresh task to process the payload
// Per TASK_WORKFLOW_2024-12-10.md: Task chaining
// ============================================================
await taskService.createTask({
role: 'product_refresh',
dispensary_id: dispensaryId,
priority: task.priority || 0,
payload: { payload_id: payloadResult.id },
});
console.log(`[PayloadFetch] Queued product_refresh task for payload #${payloadResult.id}`);
return {
success: true,
payloadId: payloadResult.id,
productCount: allProducts.length,
sizeBytes: payloadResult.sizeBytes,
};
} catch (error: unknown) {
const errorMessage = error instanceof Error ? error.message : 'Unknown error';
console.error(`[PayloadFetch] Error for dispensary ${dispensaryId}:`, errorMessage);
return {
success: false,
error: errorMessage,
};
}
}

29
backend/src/tasks/handlers/product-discovery.ts
View File

@@ -1,16 +1,31 @@
/**
* Product Discovery Handler
*
* Initial product fetch for stores that have 0 products.
* Same logic as product_resync, but for initial discovery.
* Per TASK_WORKFLOW_2024-12-10.md: Initial product fetch for newly discovered stores.
*
* Flow:
* 1. Triggered after store_discovery promotes a new dispensary
* 2. Chains to payload_fetch to get initial product data
* 3. payload_fetch chains to product_refresh for DB upsert
*
* Chaining:
* store_discovery → (newStoreIds) → product_discovery → payload_fetch → product_refresh
*/
import { TaskContext, TaskResult } from '../task-worker';
import { handleProductRefresh } from './product-refresh';
import { handlePayloadFetch } from './payload-fetch';
export async function handleProductDiscovery(ctx: TaskContext): Promise<TaskResult> {
// Product discovery is essentially the same as refresh for the first time
// The main difference is in when this task is triggered (new store vs scheduled)
console.log(`[ProductDiscovery] Starting initial product fetch for dispensary ${ctx.task.dispensary_id}`);
return handleProductRefresh(ctx);
const { task } = ctx;
const dispensaryId = task.dispensary_id;
if (!dispensaryId) {
return { success: false, error: 'No dispensary_id provided' };
}
console.log(`[ProductDiscovery] Starting initial product discovery for dispensary ${dispensaryId}`);
// Per TASK_WORKFLOW_2024-12-10.md: Chain to payload_fetch for API → disk
// payload_fetch will then chain to product_refresh for disk → DB
return handlePayloadFetch(ctx);
}

199
backend/src/tasks/handlers/product-refresh.ts
View File

@@ -1,33 +1,32 @@
/**
* Product Refresh Handler
*
* Re-crawls a store to capture price/stock changes using the GraphQL pipeline.
* Per TASK_WORKFLOW_2024-12-10.md: Processes a locally-stored payload.
*
* This handler reads from the filesystem (NOT the Dutchie API).
* The payload_fetch handler is responsible for API calls.
*
* Flow:
* 1. Load dispensary info from database
* 2. Start stealth session (fingerprint + optional proxy)
* 3. Fetch products via GraphQL (Status: 'All')
* 4. Normalize data via DutchieNormalizer
* 5. Upsert to store_products and store_product_snapshots
* 6. Track missing products (increment consecutive_misses, mark OOS at 3)
* 7. Download new product images
* 8. End session
* 1. Load payload from filesystem (by payload_id or latest for dispensary)
* 2. Normalize data via DutchieNormalizer
* 3. Upsert to store_products and store_product_snapshots
* 4. Track missing products (increment consecutive_misses, mark OOS at 3)
* 5. Download new product images
*
* Benefits of separation:
* - Retry-friendly: If this fails, re-run without re-crawling
* - Replay-able: Run against any historical payload
* - Faster: Local file read vs network call
*/
import { TaskContext, TaskResult } from '../task-worker';
import {
executeGraphQL,
startSession,
endSession,
GRAPHQL_HASHES,
DUTCHIE_CONFIG,
} from '../../platforms/dutchie';
import { DutchieNormalizer } from '../../hydration/normalizers/dutchie';
import {
upsertStoreProducts,
createStoreProductSnapshots,
downloadProductImages,
} from '../../hydration/canonical-upsert';
import { loadRawPayloadById, getLatestPayload } from '../../utils/payload-storage';
const normalizer = new DutchieNormalizer();
@@ -47,129 +46,76 @@ export async function handleProductRefresh(ctx: TaskContext): Promise<TaskResult
SELECT
id, name, platform_dispensary_id, menu_url, menu_type, city, state
FROM dispensaries
WHERE id = $1 AND crawl_enabled = true
WHERE id = $1
`, [dispensaryId]);
if (dispResult.rows.length === 0) {
return { success: false, error: `Dispensary ${dispensaryId} not found or not crawl_enabled` };
return { success: false, error: `Dispensary ${dispensaryId} not found` };
}
const dispensary = dispResult.rows[0];
const platformId = dispensary.platform_dispensary_id;
if (!platformId) {
return { success: false, error: `Dispensary ${dispensaryId} has no platform_dispensary_id` };
}
// Extract cName from menu_url
// Extract cName from menu_url for image storage context
const cNameMatch = dispensary.menu_url?.match(/\/(?:embedded-menu|dispensary)\/([^/?]+)/);
const cName = cNameMatch ? cNameMatch[1] : 'dispensary';
console.log(`[ProductResync] Starting crawl for ${dispensary.name} (ID: ${dispensaryId})`);
console.log(`[ProductResync] Platform ID: ${platformId}, cName: ${cName}`);
// ============================================================
// STEP 2: Start stealth session
// ============================================================
const session = startSession(dispensary.state || 'AZ', 'America/Phoenix');
console.log(`[ProductResync] Session started: ${session.sessionId}`);
console.log(`[ProductRefresh] Starting refresh for ${dispensary.name} (ID: ${dispensaryId})`);
await ctx.heartbeat();
// ============================================================
// STEP 3: Fetch products via GraphQL (Status: 'All')
// STEP 2: Load payload from filesystem
// Per TASK_WORKFLOW_2024-12-10.md: Read local payload, not API
// ============================================================
const allProducts: any[] = [];
let page = 0;
let totalCount = 0;
const perPage = DUTCHIE_CONFIG.perPage;
const maxPages = DUTCHIE_CONFIG.maxPages;
let payloadData: any;
let payloadId: number;
try {
while (page < maxPages) {
const variables = {
includeEnterpriseSpecials: false,
productsFilter: {
dispensaryId: platformId,
pricingType: 'rec',
Status: 'All',
types: [],
useCache: false,
isDefaultSort: true,
sortBy: 'popularSortIdx',
sortDirection: 1,
bypassOnlineThresholds: true,
isKioskMenu: false,
removeProductsBelowOptionThresholds: false,
},
page,
perPage,
};
// Check if specific payload_id was provided (from task chaining)
const taskPayload = task.payload as { payload_id?: number } | null;
console.log(`[ProductResync] Fetching page ${page + 1}...`);
const result = await executeGraphQL(
'FilteredProducts',
variables,
GRAPHQL_HASHES.FilteredProducts,
{ cName, maxRetries: 3 }
);
const data = result?.data?.filteredProducts;
if (!data || !data.products) {
if (page === 0) {
throw new Error('No product data returned from GraphQL');
}
break;
}
const products = data.products;
allProducts.push(...products);
if (page === 0) {
totalCount = data.queryInfo?.totalCount || products.length;
console.log(`[ProductResync] Total products reported: ${totalCount}`);
}
if (allProducts.length >= totalCount || products.length < perPage) {
break;
}
page++;
if (page < maxPages) {
await new Promise(r => setTimeout(r, DUTCHIE_CONFIG.pageDelayMs));
}
if (page % 5 === 0) {
await ctx.heartbeat();
}
if (taskPayload?.payload_id) {
// Load specific payload (from payload_fetch chaining)
const result = await loadRawPayloadById(pool, taskPayload.payload_id);
if (!result) {
return { success: false, error: `Payload ${taskPayload.payload_id} not found` };
}
console.log(`[ProductResync] Fetched ${allProducts.length} products in ${page + 1} pages`);
} finally {
endSession();
payloadData = result.payload;
payloadId = result.metadata.id;
console.log(`[ProductRefresh] Loaded specific payload #${payloadId}`);
} else {
// Load latest payload for this dispensary
const result = await getLatestPayload(pool, dispensaryId);
if (!result) {
return { success: false, error: `No payload found for dispensary ${dispensaryId}` };
}
payloadData = result.payload;
payloadId = result.metadata.id;
console.log(`[ProductRefresh] Loaded latest payload #${payloadId} (${result.metadata.fetchedAt})`);
}
const allProducts = payloadData.products || [];
if (allProducts.length === 0) {
return {
success: false,
error: 'No products returned from GraphQL',
error: 'Payload contains no products',
payloadId,
productsProcessed: 0,
};
}
console.log(`[ProductRefresh] Processing ${allProducts.length} products from payload #${payloadId}`);
await ctx.heartbeat();
// ============================================================
// STEP 4: Normalize data
// STEP 3: Normalize data
// ============================================================
console.log(`[ProductResync] Normalizing ${allProducts.length} products...`);
console.log(`[ProductRefresh] Normalizing ${allProducts.length} products...`);
// Build RawPayload for the normalizer
const rawPayload = {
id: `resync-${dispensaryId}-${Date.now()}`,
id: `refresh-${dispensaryId}-${Date.now()}`,
dispensary_id: dispensaryId,
crawl_run_id: null,
platform: 'dutchie',
@@ -189,25 +135,26 @@ export async function handleProductRefresh(ctx: TaskContext): Promise<TaskResult
const normalizationResult = normalizer.normalize(rawPayload);
if (normalizationResult.errors.length > 0) {
console.warn(`[ProductResync] Normalization warnings: ${normalizationResult.errors.map(e => e.message).join(', ')}`);
console.warn(`[ProductRefresh] Normalization warnings: ${normalizationResult.errors.map(e => e.message).join(', ')}`);
}
if (normalizationResult.products.length === 0) {
return {
success: false,
error: 'Normalization produced no products',
payloadId,
productsProcessed: 0,
};
}
console.log(`[ProductResync] Normalized ${normalizationResult.products.length} products`);
console.log(`[ProductRefresh] Normalized ${normalizationResult.products.length} products`);
await ctx.heartbeat();
// ============================================================
// STEP 5: Upsert to canonical tables
// STEP 4: Upsert to canonical tables
// ============================================================
console.log(`[ProductResync] Upserting to store_products...`);
console.log(`[ProductRefresh] Upserting to store_products...`);
const upsertResult = await upsertStoreProducts(
pool,
@@ -216,12 +163,12 @@ export async function handleProductRefresh(ctx: TaskContext): Promise<TaskResult
normalizationResult.availability
);
console.log(`[ProductResync] Upserted: ${upsertResult.upserted} (${upsertResult.new} new, ${upsertResult.updated} updated)`);
console.log(`[ProductRefresh] Upserted: ${upsertResult.upserted} (${upsertResult.new} new, ${upsertResult.updated} updated)`);
await ctx.heartbeat();
// Create snapshots
console.log(`[ProductResync] Creating snapshots...`);
console.log(`[ProductRefresh] Creating snapshots...`);
const snapshotsResult = await createStoreProductSnapshots(
pool,
@@ -232,12 +179,12 @@ export async function handleProductRefresh(ctx: TaskContext): Promise<TaskResult
null // No crawl_run_id in new system
);
console.log(`[ProductResync] Created ${snapshotsResult.created} snapshots`);
console.log(`[ProductRefresh] Created ${snapshotsResult.created} snapshots`);
await ctx.heartbeat();
// ============================================================
// STEP 6: Track missing products (consecutive_misses logic)
// STEP 5: Track missing products (consecutive_misses logic)
// - Products in feed: reset consecutive_misses to 0
// - Products not in feed: increment consecutive_misses
// - At 3 consecutive misses: mark as OOS
@@ -270,7 +217,7 @@ export async function handleProductRefresh(ctx: TaskContext): Promise<TaskResult
const incrementedCount = incrementResult.rowCount || 0;
if (incrementedCount > 0) {
console.log(`[ProductResync] Incremented consecutive_misses for ${incrementedCount} products`);
console.log(`[ProductRefresh] Incremented consecutive_misses for ${incrementedCount} products`);
}
// Mark as OOS any products that hit 3 consecutive misses
@@ -286,16 +233,16 @@ export async function handleProductRefresh(ctx: TaskContext): Promise<TaskResult
const markedOosCount = oosResult.rowCount || 0;
if (markedOosCount > 0) {
console.log(`[ProductResync] Marked ${markedOosCount} products as OOS (3+ consecutive misses)`);
console.log(`[ProductRefresh] Marked ${markedOosCount} products as OOS (3+ consecutive misses)`);
}
await ctx.heartbeat();
// ============================================================
// STEP 7: Download images for new products
// STEP 6: Download images for new products
// ============================================================
if (upsertResult.productsNeedingImages.length > 0) {
console.log(`[ProductResync] Downloading images for ${upsertResult.productsNeedingImages.length} products...`);
console.log(`[ProductRefresh] Downloading images for ${upsertResult.productsNeedingImages.length} products...`);
try {
const dispensaryContext = {
@@ -309,12 +256,12 @@ export async function handleProductRefresh(ctx: TaskContext): Promise<TaskResult
);
} catch (imgError: any) {
// Image download errors shouldn't fail the whole task
console.warn(`[ProductResync] Image download error (non-fatal): ${imgError.message}`);
console.warn(`[ProductRefresh] Image download error (non-fatal): ${imgError.message}`);
}
}
// ============================================================
// STEP 8: Update dispensary last_crawl_at
// STEP 7: Update dispensary last_crawl_at
// ============================================================
await pool.query(`
UPDATE dispensaries
@@ -322,10 +269,20 @@ export async function handleProductRefresh(ctx: TaskContext): Promise<TaskResult
WHERE id = $1
`, [dispensaryId]);
console.log(`[ProductResync] Completed ${dispensary.name}`);
// ============================================================
// STEP 8: Mark payload as processed
// ============================================================
await pool.query(`
UPDATE raw_crawl_payloads
SET processed_at = NOW()
WHERE id = $1
`, [payloadId]);
console.log(`[ProductRefresh] Completed ${dispensary.name}`);
return {
success: true,
payloadId,
productsProcessed: normalizationResult.products.length,
snapshotsCreated: snapshotsResult.created,
newProducts: upsertResult.new,
@@ -335,7 +292,7 @@ export async function handleProductRefresh(ctx: TaskContext): Promise<TaskResult
} catch (error: unknown) {
const errorMessage = error instanceof Error ? error.message : 'Unknown error';
console.error(`[ProductResync] Error for dispensary ${dispensaryId}:`, errorMessage);
console.error(`[ProductRefresh] Error for dispensary ${dispensaryId}:`, errorMessage);
return {
success: false,
error: errorMessage,

30
backend/src/tasks/handlers/store-discovery.ts
View File

@@ -1,8 +1,16 @@
/**
* Store Discovery Handler
*
* Discovers new stores by crawling location APIs and adding them
* to discovery_locations table.
* Per TASK_WORKFLOW_2024-12-10.md: Discovers new stores and returns their IDs for task chaining.
*
* Flow:
* 1. For each active state, run Dutchie discovery
* 2. Discover locations via GraphQL
* 3. Auto-promote valid locations to dispensaries table
* 4. Return newStoreIds[] for chaining to payload_fetch
*
* Chaining:
* store_discovery → (returns newStoreIds) → payload_fetch → product_refresh
*/
import { TaskContext, TaskResult } from '../task-worker';
@@ -10,7 +18,7 @@ import { discoverState } from '../../discovery';
export async function handleStoreDiscovery(ctx: TaskContext): Promise<TaskResult> {
const { pool, task } = ctx;
const platform = task.platform || 'default';
const platform = task.platform || 'dutchie';
console.log(`[StoreDiscovery] Starting discovery for platform: ${platform}`);
@@ -22,11 +30,13 @@ export async function handleStoreDiscovery(ctx: TaskContext): Promise<TaskResult
const stateCodes = statesResult.rows.map(r => r.code);
if (stateCodes.length === 0) {
return { success: true, storesDiscovered: 0, message: 'No active states to discover' };
return { success: true, storesDiscovered: 0, newStoreIds: [], message: 'No active states to discover' };
}
let totalDiscovered = 0;
let totalPromoted = 0;
// Per TASK_WORKFLOW_2024-12-10.md: Collect all new store IDs for task chaining
const allNewStoreIds: number[] = [];
// Run discovery for each state
for (const stateCode of stateCodes) {
@@ -39,6 +49,13 @@ export async function handleStoreDiscovery(ctx: TaskContext): Promise<TaskResult
const result = await discoverState(pool, stateCode);
totalDiscovered += result.totalLocationsFound || 0;
totalPromoted += result.totalLocationsUpserted || 0;
// Per TASK_WORKFLOW_2024-12-10.md: Collect new IDs for chaining
if (result.newDispensaryIds && result.newDispensaryIds.length > 0) {
allNewStoreIds.push(...result.newDispensaryIds);
console.log(`[StoreDiscovery] ${stateCode}: ${result.newDispensaryIds.length} new stores`);
}
console.log(`[StoreDiscovery] ${stateCode}: found ${result.totalLocationsFound}, upserted ${result.totalLocationsUpserted}`);
} catch (error: unknown) {
const errorMessage = error instanceof Error ? error.message : 'Unknown error';
@@ -47,13 +64,15 @@ export async function handleStoreDiscovery(ctx: TaskContext): Promise<TaskResult
}
}
console.log(`[StoreDiscovery] Complete: ${totalDiscovered} discovered, ${totalPromoted} promoted`);
console.log(`[StoreDiscovery] Complete: ${totalDiscovered} discovered, ${totalPromoted} promoted, ${allNewStoreIds.length} new stores`);
return {
success: true,
storesDiscovered: totalDiscovered,
storesPromoted: totalPromoted,
statesProcessed: stateCodes.length,
// Per TASK_WORKFLOW_2024-12-10.md: Return new IDs for task chaining
newStoreIds: allNewStoreIds,
};
} catch (error: unknown) {
const errorMessage = error instanceof Error ? error.message : 'Unknown error';
@@ -61,6 +80,7 @@ export async function handleStoreDiscovery(ctx: TaskContext): Promise<TaskResult
return {
success: false,
error: errorMessage,
newStoreIds: [],
};
}
}

44
backend/src/tasks/task-service.ts
View File

@@ -21,11 +21,15 @@ async function tableExists(tableName: string): Promise<boolean> {
return result.rows[0].exists;
}
// Per TASK_WORKFLOW_2024-12-10.md: Task roles
// payload_fetch: Hits Dutchie API, saves raw payload to filesystem
// product_refresh: Reads local payload, normalizes, upserts to DB
export type TaskRole =
| 'store_discovery'
| 'entry_point_discovery'
| 'product_discovery'
| 'product_refresh'
| 'payload_fetch' // NEW: Fetches from API, saves to disk
| 'product_refresh' // CHANGED: Now reads from local payload
| 'analytics_refresh';
export type TaskStatus =
@@ -55,6 +59,7 @@ export interface WorkerTask {
error_message: string | null;
retry_count: number;
max_retries: number;
payload: Record<string, unknown> | null; // Per TASK_WORKFLOW_2024-12-10.md: Task chaining data
created_at: Date;
updated_at: Date;
}
@@ -65,6 +70,7 @@ export interface CreateTaskParams {
platform?: string;
priority?: number;
scheduled_for?: Date;
payload?: Record<string, unknown>; // Per TASK_WORKFLOW_2024-12-10.md: For task chaining data
}
export interface CapacityMetrics {
@@ -96,8 +102,8 @@ class TaskService {
*/
async createTask(params: CreateTaskParams): Promise<WorkerTask> {
const result = await pool.query(
`INSERT INTO worker_tasks (role, dispensary_id, platform, priority, scheduled_for)
VALUES ($1, $2, $3, $4, $5)
`INSERT INTO worker_tasks (role, dispensary_id, platform, priority, scheduled_for, payload)
VALUES ($1, $2, $3, $4, $5, $6)
RETURNING *`,
[
params.role,
@@ -105,6 +111,7 @@ class TaskService {
params.platform ?? null,
params.priority ?? 0,
params.scheduled_for ?? null,
params.payload ? JSON.stringify(params.payload) : null,
]
);
return result.rows[0] as WorkerTask;
@@ -401,6 +408,17 @@ class TaskService {
/**
* Chain next task after completion
* Called automatically when a task completes successfully
*
* Per TASK_WORKFLOW_2024-12-10.md: Task chaining flow:
*
* Discovery flow (new stores):
* store_discovery → product_discovery → payload_fetch → product_refresh
*
* Scheduled flow (existing stores):
* payload_fetch → product_refresh
*
* Note: entry_point_discovery is deprecated since platform_dispensary_id
* is now resolved during store promotion.
*/
async chainNextTask(completedTask: WorkerTask): Promise<WorkerTask | null> {
if (completedTask.status !== 'completed') {
@@ -409,12 +427,14 @@ class TaskService {
switch (completedTask.role) {
case 'store_discovery': {
// New stores discovered -> create entry_point_discovery tasks
// Per TASK_WORKFLOW_2024-12-10.md: New stores discovered -> create product_discovery tasks
// Skip entry_point_discovery since platform_dispensary_id is set during promotion
const newStoreIds = (completedTask.result as { newStoreIds?: number[] })?.newStoreIds;
if (newStoreIds && newStoreIds.length > 0) {
console.log(`[TaskService] Chaining ${newStoreIds.length} product_discovery tasks for new stores`);
for (const storeId of newStoreIds) {
await this.createTask({
role: 'entry_point_discovery',
role: 'product_discovery',
dispensary_id: storeId,
platform: completedTask.platform ?? undefined,
priority: 10, // High priority for new stores
@@ -425,7 +445,8 @@ class TaskService {
}
case 'entry_point_discovery': {
// Entry point resolved -> create product_discovery task
// DEPRECATED: Entry point resolution now happens during store promotion
// Kept for backward compatibility with any in-flight tasks
const success = (completedTask.result as { success?: boolean })?.success;
if (success && completedTask.dispensary_id) {
return this.createTask({
@@ -439,8 +460,15 @@ class TaskService {
}
case 'product_discovery': {
// Product discovery done -> store is now ready for regular resync
// No immediate chaining needed; will be picked up by daily batch generation
// Per TASK_WORKFLOW_2024-12-10.md: Product discovery chains internally to payload_fetch
// No external chaining needed - handleProductDiscovery calls handlePayloadFetch directly
break;
}
case 'payload_fetch': {
// Per TASK_WORKFLOW_2024-12-10.md: payload_fetch chains to product_refresh
// This is handled internally by the payload_fetch handler via taskService.createTask
// No external chaining needed here
break;
}
}

12
backend/src/tasks/task-worker.ts
View File

@@ -52,6 +52,8 @@ import { CrawlRotator } from '../services/crawl-rotator';
import { setCrawlRotator } from '../platforms/dutchie';
// Task handlers by role
// Per TASK_WORKFLOW_2024-12-10.md: payload_fetch and product_refresh are now separate
import { handlePayloadFetch } from './handlers/payload-fetch';
import { handleProductRefresh } from './handlers/product-refresh';
import { handleProductDiscovery } from './handlers/product-discovery';
import { handleStoreDiscovery } from './handlers/store-discovery';
@@ -80,8 +82,12 @@ export interface TaskResult {
type TaskHandler = (ctx: TaskContext) => Promise<TaskResult>;
// Per TASK_WORKFLOW_2024-12-10.md: Handler registry
// payload_fetch: Fetches from Dutchie API, saves to disk, chains to product_refresh
// product_refresh: Reads local payload, normalizes, upserts to DB
const TASK_HANDLERS: Record<TaskRole, TaskHandler> = {
product_refresh: handleProductRefresh,
payload_fetch: handlePayloadFetch, // NEW: API fetch -> disk
product_refresh: handleProductRefresh, // CHANGED: disk -> DB
product_discovery: handleProductDiscovery,
store_discovery: handleStoreDiscovery,
entry_point_discovery: handleEntryPointDiscovery,
@@ -414,11 +420,13 @@ export class TaskWorker {
async function main(): Promise<void> {
const role = process.env.WORKER_ROLE as TaskRole | undefined;
// Per TASK_WORKFLOW_2024-12-10.md: Valid task roles
const validRoles: TaskRole[] = [
'store_discovery',
'entry_point_discovery',
'product_discovery',
'product_refresh',
'payload_fetch', // NEW: Fetches from API, saves to disk
'product_refresh', // CHANGED: Reads from disk, processes to DB
'analytics_refresh',
];

49
backend/src/types/user-agents.d.ts vendored Normal file
View File

@@ -0,0 +1,49 @@
/**
* Type declarations for user-agents npm package
* Per workflow-12102025.md: Used for realistic UA generation with market-share weighting
*/
declare module 'user-agents' {
interface UserAgentData {
userAgent: string;
platform: string;
screenWidth: number;
screenHeight: number;
viewportWidth: number;
viewportHeight: number;
deviceCategory: 'desktop' | 'mobile' | 'tablet';
appName: string;
connection?: {
downlink: number;
effectiveType: string;
rtt: number;
};
}
interface UserAgentOptions {
deviceCategory?: 'desktop' | 'mobile' | 'tablet';
platform?: RegExp | string;
screenWidth?: RegExp | { min?: number; max?: number };
screenHeight?: RegExp | { min?: number; max?: number };
}
interface UserAgentInstance {
data: UserAgentData;
toString(): string;
random(): UserAgentInstance;
}
class UserAgent {
constructor(options?: UserAgentOptions | UserAgentOptions[]);
data: UserAgentData;
toString(): string;
random(): UserAgentInstance;
}
// Make it callable
interface UserAgent {
(): UserAgentInstance;
}
export default UserAgent;
}

406
backend/src/utils/payload-storage.ts Normal file
View File

@@ -0,0 +1,406 @@
/**
* Payload Storage Utility
*
* Per TASK_WORKFLOW_2024-12-10.md: Store raw GraphQL payloads for historical analysis.
*
* Design Pattern: Metadata/Payload Separation
* - Metadata in PostgreSQL (raw_crawl_payloads table): Small, indexed, queryable
* - Payload on filesystem: Gzipped JSON at storage_path
*
* Storage structure:
* /storage/payloads/{year}/{month}/{day}/store_{dispensary_id}_{timestamp}.json.gz
*
* Benefits:
* - Compare any two crawls to see what changed
* - Replay/re-normalize historical data if logic changes
* - Debug issues by seeing exactly what the API returned
* - DB stays small, backups stay fast
* - ~90% compression (1.5MB -> 150KB per crawl)
*/
import * as fs from 'fs';
import * as path from 'path';
import * as zlib from 'zlib';
import { promisify } from 'util';
import { Pool } from 'pg';
import * as crypto from 'crypto';
const gzip = promisify(zlib.gzip);
const gunzip = promisify(zlib.gunzip);
// Base path for payload storage (matches image storage pattern)
const PAYLOAD_BASE_PATH = process.env.PAYLOAD_STORAGE_PATH || './storage/payloads';
/**
* Result from saving a payload
*/
export interface SavePayloadResult {
id: number;
storagePath: string;
sizeBytes: number;
sizeBytesRaw: number;
checksum: string;
}
/**
* Result from loading a payload
*/
export interface LoadPayloadResult {
payload: any;
metadata: {
id: number;
dispensaryId: number;
crawlRunId: number | null;
productCount: number;
fetchedAt: Date;
storagePath: string;
};
}
/**
* Generate storage path for a payload
*
* Format: /storage/payloads/{year}/{month}/{day}/store_{dispensary_id}_{timestamp}.json.gz
*/
function generateStoragePath(dispensaryId: number, timestamp: Date): string {
const year = timestamp.getFullYear();
const month = String(timestamp.getMonth() + 1).padStart(2, '0');
const day = String(timestamp.getDate()).padStart(2, '0');
const ts = timestamp.getTime();
return path.join(
PAYLOAD_BASE_PATH,
String(year),
month,
day,
`store_${dispensaryId}_${ts}.json.gz`
);
}
/**
* Ensure directory exists for a file path
*/
async function ensureDir(filePath: string): Promise<void> {
const dir = path.dirname(filePath);
await fs.promises.mkdir(dir, { recursive: true });
}
/**
* Calculate SHA256 checksum of data
*/
function calculateChecksum(data: Buffer): string {
return crypto.createHash('sha256').update(data).digest('hex');
}
/**
* Save a raw crawl payload to filesystem and record metadata in DB
*
* @param pool - Database connection pool
* @param dispensaryId - ID of the dispensary
* @param payload - Raw JSON payload from GraphQL
* @param crawlRunId - Optional crawl_run ID for linking
* @param productCount - Number of products in payload
* @returns SavePayloadResult with file info and DB record ID
*/
export async function saveRawPayload(
pool: Pool,
dispensaryId: number,
payload: any,
crawlRunId: number | null = null,
productCount: number = 0
): Promise<SavePayloadResult> {
const timestamp = new Date();
const storagePath = generateStoragePath(dispensaryId, timestamp);
// Serialize and compress
const jsonStr = JSON.stringify(payload);
const rawSize = Buffer.byteLength(jsonStr, 'utf8');
const compressed = await gzip(Buffer.from(jsonStr, 'utf8'));
const compressedSize = compressed.length;
const checksum = calculateChecksum(compressed);
// Write to filesystem
await ensureDir(storagePath);
await fs.promises.writeFile(storagePath, compressed);
// Record metadata in DB
const result = await pool.query(`
INSERT INTO raw_crawl_payloads (
crawl_run_id,
dispensary_id,
storage_path,
product_count,
size_bytes,
size_bytes_raw,
fetched_at,
checksum_sha256
) VALUES ($1, $2, $3, $4, $5, $6, $7, $8)
RETURNING id
`, [
crawlRunId,
dispensaryId,
storagePath,
productCount,
compressedSize,
rawSize,
timestamp,
checksum
]);
console.log(`[PayloadStorage] Saved payload for store ${dispensaryId}: ${storagePath} (${(compressedSize / 1024).toFixed(1)}KB compressed, ${(rawSize / 1024).toFixed(1)}KB raw)`);
return {
id: result.rows[0].id,
storagePath,
sizeBytes: compressedSize,
sizeBytesRaw: rawSize,
checksum
};
}
/**
* Load a raw payload from filesystem by metadata ID
*
* @param pool - Database connection pool
* @param payloadId - ID from raw_crawl_payloads table
* @returns LoadPayloadResult with parsed payload and metadata
*/
export async function loadRawPayloadById(
pool: Pool,
payloadId: number
): Promise<LoadPayloadResult | null> {
const result = await pool.query(`
SELECT id, dispensary_id, crawl_run_id, storage_path, product_count, fetched_at
FROM raw_crawl_payloads
WHERE id = $1
`, [payloadId]);
if (result.rows.length === 0) {
return null;
}
const row = result.rows[0];
const payload = await loadPayloadFromPath(row.storage_path);
return {
payload,
metadata: {
id: row.id,
dispensaryId: row.dispensary_id,
crawlRunId: row.crawl_run_id,
productCount: row.product_count,
fetchedAt: row.fetched_at,
storagePath: row.storage_path
}
};
}
/**
* Load a raw payload directly from filesystem path
*
* @param storagePath - Path to gzipped JSON file
* @returns Parsed JSON payload
*/
export async function loadPayloadFromPath(storagePath: string): Promise<any> {
const compressed = await fs.promises.readFile(storagePath);
const decompressed = await gunzip(compressed);
return JSON.parse(decompressed.toString('utf8'));
}
/**
* Get the latest payload for a dispensary
*
* @param pool - Database connection pool
* @param dispensaryId - ID of the dispensary
* @returns LoadPayloadResult or null if none exists
*/
export async function getLatestPayload(
pool: Pool,
dispensaryId: number
): Promise<LoadPayloadResult | null> {
const result = await pool.query(`
SELECT id, dispensary_id, crawl_run_id, storage_path, product_count, fetched_at
FROM raw_crawl_payloads
WHERE dispensary_id = $1
ORDER BY fetched_at DESC
LIMIT 1
`, [dispensaryId]);
if (result.rows.length === 0) {
return null;
}
const row = result.rows[0];
const payload = await loadPayloadFromPath(row.storage_path);
return {
payload,
metadata: {
id: row.id,
dispensaryId: row.dispensary_id,
crawlRunId: row.crawl_run_id,
productCount: row.product_count,
fetchedAt: row.fetched_at,
storagePath: row.storage_path
}
};
}
/**
* Get two payloads for comparison (latest and previous, or by IDs)
*
* @param pool - Database connection pool
* @param dispensaryId - ID of the dispensary
* @param limit - Number of recent payloads to retrieve (default 2)
* @returns Array of LoadPayloadResult, most recent first
*/
export async function getRecentPayloads(
pool: Pool,
dispensaryId: number,
limit: number = 2
): Promise<LoadPayloadResult[]> {
const result = await pool.query(`
SELECT id, dispensary_id, crawl_run_id, storage_path, product_count, fetched_at
FROM raw_crawl_payloads
WHERE dispensary_id = $1
ORDER BY fetched_at DESC
LIMIT $2
`, [dispensaryId, limit]);
const payloads: LoadPayloadResult[] = [];
for (const row of result.rows) {
const payload = await loadPayloadFromPath(row.storage_path);
payloads.push({
payload,
metadata: {
id: row.id,
dispensaryId: row.dispensary_id,
crawlRunId: row.crawl_run_id,
productCount: row.product_count,
fetchedAt: row.fetched_at,
storagePath: row.storage_path
}
});
}
return payloads;
}
/**
* List payload metadata without loading files (for browsing/pagination)
*
* @param pool - Database connection pool
* @param options - Query options
* @returns Array of metadata rows
*/
export async function listPayloadMetadata(
pool: Pool,
options: {
dispensaryId?: number;
startDate?: Date;
endDate?: Date;
limit?: number;
offset?: number;
} = {}
): Promise<Array<{
id: number;
dispensaryId: number;
crawlRunId: number | null;
storagePath: string;
productCount: number;
sizeBytes: number;
sizeBytesRaw: number;
fetchedAt: Date;
}>> {
const conditions: string[] = [];
const params: any[] = [];
let paramIndex = 1;
if (options.dispensaryId) {
conditions.push(`dispensary_id = $${paramIndex++}`);
params.push(options.dispensaryId);
}
if (options.startDate) {
conditions.push(`fetched_at >= $${paramIndex++}`);
params.push(options.startDate);
}
if (options.endDate) {
conditions.push(`fetched_at <= $${paramIndex++}`);
params.push(options.endDate);
}
const whereClause = conditions.length > 0 ? `WHERE ${conditions.join(' AND ')}` : '';
const limit = options.limit || 50;
const offset = options.offset || 0;
params.push(limit, offset);
const result = await pool.query(`
SELECT
id,
dispensary_id,
crawl_run_id,
storage_path,
product_count,
size_bytes,
size_bytes_raw,
fetched_at
FROM raw_crawl_payloads
${whereClause}
ORDER BY fetched_at DESC
LIMIT $${paramIndex++} OFFSET $${paramIndex}
`, params);
return result.rows.map(row => ({
id: row.id,
dispensaryId: row.dispensary_id,
crawlRunId: row.crawl_run_id,
storagePath: row.storage_path,
productCount: row.product_count,
sizeBytes: row.size_bytes,
sizeBytesRaw: row.size_bytes_raw,
fetchedAt: row.fetched_at
}));
}
/**
* Delete old payloads (for retention policy)
*
* @param pool - Database connection pool
* @param olderThan - Delete payloads older than this date
* @returns Number of payloads deleted
*/
export async function deleteOldPayloads(
pool: Pool,
olderThan: Date
): Promise<number> {
// Get paths first
const result = await pool.query(`
SELECT id, storage_path FROM raw_crawl_payloads
WHERE fetched_at < $1
`, [olderThan]);
// Delete files
for (const row of result.rows) {
try {
await fs.promises.unlink(row.storage_path);
} catch (err: any) {
if (err.code !== 'ENOENT') {
console.warn(`[PayloadStorage] Failed to delete ${row.storage_path}: ${err.message}`);
}
}
}
// Delete DB records
await pool.query(`
DELETE FROM raw_crawl_payloads
WHERE fetched_at < $1
`, [olderThan]);
console.log(`[PayloadStorage] Deleted ${result.rows.length} payloads older than ${olderThan.toISOString()}`);
return result.rows.length;
}