feat(tasks): Refactor task workflow with payload/refresh separation

Major changes:
- Split crawl into payload_fetch (API → disk) and product_refresh (disk → DB)
- Add task chaining: store_discovery → product_discovery → payload_fetch → product_refresh
- Add payload storage utilities for gzipped JSON on filesystem
- Add /api/payloads endpoints for payload access and diffing
- Add DB-driven TaskScheduler with schedule persistence
- Track newDispensaryIds through discovery promotion for chaining
- Add stealth improvements: HTTP fingerprinting, proxy rotation enhancements
- Add Workers dashboard K8s scaling controls

New files:
- src/tasks/handlers/payload-fetch.ts - Fetches from API, saves to disk
- src/services/task-scheduler.ts - DB-driven schedule management
- src/utils/payload-storage.ts - Payload save/load utilities
- src/routes/payloads.ts - Payload API endpoints
- src/services/http-fingerprint.ts - Browser fingerprint generation
- docs/TASK_WORKFLOW_2024-12-10.md - Complete workflow documentation

Migrations:
- 078: Proxy consecutive 403 tracking
- 079: task_schedules table
- 080: raw_crawl_payloads table
- 081: payload column and last_fetch_at

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>

backend/src/discovery/discovery-crawler.ts

@@ -172,6 +172,9 @@ export async function runFullDiscovery(
     console.log(`Errors: ${totalErrors}`);
   }
 
+  // Per TASK_WORKFLOW_2024-12-10.md: Track new dispensary IDs for task chaining
+  let newDispensaryIds: number[] = [];
+
   // Step 4: Auto-validate and promote discovered locations
   if (!dryRun && totalLocationsUpserted > 0) {
     console.log('\n[Discovery] Step 4: Auto-promoting discovered locations...');
@@ -180,6 +183,13 @@ export async function runFullDiscovery(
     console.log(`  Created: ${promotionResult.created} new dispensaries`);
     console.log(`  Updated: ${promotionResult.updated} existing dispensaries`);
     console.log(`  Rejected: ${promotionResult.rejected} (validation failed)`);
+
+    // Per TASK_WORKFLOW_2024-12-10.md: Capture new IDs for task chaining
+    newDispensaryIds = promotionResult.newDispensaryIds;
+    if (newDispensaryIds.length > 0) {
+      console.log(`  New store IDs for crawl: [${newDispensaryIds.join(', ')}]`);
+    }
+
     if (promotionResult.rejectedRecords.length > 0) {
       console.log(`  Rejection reasons:`);
       promotionResult.rejectedRecords.slice(0, 5).forEach(r => {
@@ -214,6 +224,8 @@ export async function runFullDiscovery(
     totalLocationsFound,
     totalLocationsUpserted,
     durationMs,
+    // Per TASK_WORKFLOW_2024-12-10.md: Return new IDs for task chaining
+    newDispensaryIds,
   };
 }
 

backend/src/discovery/promotion.ts

@@ -127,6 +127,8 @@ export interface PromotionSummary {
     errors: string[];
   }>;
   durationMs: number;
+  // Per TASK_WORKFLOW_2024-12-10.md: Track new dispensary IDs for task chaining
+  newDispensaryIds: number[];
 }
 
 /**
@@ -469,6 +471,8 @@ export async function promoteDiscoveredLocations(
 
   const results: PromotionResult[] = [];
   const rejectedRecords: PromotionSummary['rejectedRecords'] = [];
+  // Per TASK_WORKFLOW_2024-12-10.md: Track new dispensary IDs for task chaining
+  const newDispensaryIds: number[] = [];
   let created = 0;
   let updated = 0;
   let skipped = 0;
@@ -525,6 +529,8 @@ export async function promoteDiscoveredLocations(
 
       if (promotionResult.action === 'created') {
         created++;
+        // Per TASK_WORKFLOW_2024-12-10.md: Track new IDs for task chaining
+        newDispensaryIds.push(promotionResult.dispensaryId);
       } else {
         updated++;
       }
@@ -548,6 +554,8 @@ export async function promoteDiscoveredLocations(
     results,
     rejectedRecords,
     durationMs: Date.now() - startTime,
+    // Per TASK_WORKFLOW_2024-12-10.md: Return new IDs for task chaining
+    newDispensaryIds,
   };
 }
 

backend/src/discovery/types.ts

@@ -211,6 +211,8 @@ export interface FullDiscoveryResult {
   totalLocationsFound: number;
   totalLocationsUpserted: number;
   durationMs: number;
+  // Per TASK_WORKFLOW_2024-12-10.md: Track new dispensary IDs for task chaining
+  newDispensaryIds?: number[];
 }
 
 // ============================================================