4949b22457
Major changes: - Split crawl into payload_fetch (API → disk) and product_refresh (disk → DB) - Add task chaining: store_discovery → product_discovery → payload_fetch → product_refresh - Add payload storage utilities for gzipped JSON on filesystem - Add /api/payloads endpoints for payload access and diffing - Add DB-driven TaskScheduler with schedule persistence - Track newDispensaryIds through discovery promotion for chaining - Add stealth improvements: HTTP fingerprinting, proxy rotation enhancements - Add Workers dashboard K8s scaling controls New files: - src/tasks/handlers/payload-fetch.ts - Fetches from API, saves to disk - src/services/task-scheduler.ts - DB-driven schedule management - src/utils/payload-storage.ts - Payload save/load utilities - src/routes/payloads.ts - Payload API endpoints - src/services/http-fingerprint.ts - Browser fingerprint generation - docs/TASK_WORKFLOW_2024-12-10.md - Complete workflow documentation Migrations: - 078: Proxy consecutive 403 tracking - 079: task_schedules table - 080: raw_crawl_payloads table - 081: payload column and last_fetch_at 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
39 lines
1.4 KiB
TypeScript
Executable File
39 lines
1.4 KiB
TypeScript
Executable File
/**
|
|
* LEGACY SCHEDULER - DEPRECATED 2024-12-10
|
|
*
|
|
* DO NOT USE THIS FILE.
|
|
*
|
|
* Per TASK_WORKFLOW_2024-12-10.md:
|
|
* This node-cron scheduler has been replaced by the database-driven
|
|
* task scheduler in src/services/task-scheduler.ts
|
|
*
|
|
* The new scheduler:
|
|
* - Stores schedules in PostgreSQL (survives restarts)
|
|
* - Uses SELECT FOR UPDATE SKIP LOCKED (multi-replica safe)
|
|
* - Creates tasks in worker_tasks table (processed by task-worker.ts)
|
|
*
|
|
* This file is kept for reference only. All exports are no-ops.
|
|
* Legacy code has been removed - see git history for original implementation.
|
|
*/
|
|
|
|
// 2024-12-10: All functions are now no-ops
|
|
export async function startScheduler(): Promise<void> {
|
|
console.warn('[DEPRECATED] startScheduler() called - use taskScheduler from task-scheduler.ts instead');
|
|
}
|
|
|
|
export function stopScheduler(): void {
|
|
console.warn('[DEPRECATED] stopScheduler() called - use taskScheduler from task-scheduler.ts instead');
|
|
}
|
|
|
|
export async function restartScheduler(): Promise<void> {
|
|
console.warn('[DEPRECATED] restartScheduler() called - use taskScheduler from task-scheduler.ts instead');
|
|
}
|
|
|
|
export async function triggerStoreScrape(_storeId: number): Promise<void> {
|
|
console.warn('[DEPRECATED] triggerStoreScrape() called - use taskService.createTask() instead');
|
|
}
|
|
|
|
export async function triggerAllStoresScrape(): Promise<void> {
|
|
console.warn('[DEPRECATED] triggerAllStoresScrape() called - use taskScheduler.triggerSchedule() instead');
|
|
}
|