feat(tasks): Add unified task-based worker architecture
Replace fragmented job systems (job_schedules, dispensary_crawl_jobs, SyncOrchestrator) with a single unified task queue: - Add worker_tasks table with atomic task claiming via SELECT FOR UPDATE SKIP LOCKED - Add TaskService for CRUD, claiming, and capacity metrics - Add TaskWorker with role-based handlers (resync, discovery, analytics) - Add /api/tasks endpoints for management and migration from legacy systems - Add TasksDashboard UI and integrate task counts into main dashboard - Add comprehensive documentation Task roles: store_discovery, entry_point_discovery, product_discovery, product_resync, analytics_refresh Run workers with: WORKER_ROLE=product_resync npx tsx src/tasks/task-worker.ts 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
67
backend/src/tasks/handlers/store-discovery.ts
Normal file
67
backend/src/tasks/handlers/store-discovery.ts
Normal file
@@ -0,0 +1,67 @@
|
||||
/**
|
||||
* Store Discovery Handler
|
||||
*
|
||||
* Discovers new stores on a platform (e.g., Dutchie) by crawling
|
||||
* location APIs and adding them to dutchie_discovery_locations.
|
||||
*/
|
||||
|
||||
import { TaskContext, TaskResult } from '../task-worker';
|
||||
import { DiscoveryCrawler } from '../../discovery/discovery-crawler';
|
||||
|
||||
export async function handleStoreDiscovery(ctx: TaskContext): Promise<TaskResult> {
|
||||
const { pool, task } = ctx;
|
||||
const platform = task.platform || 'dutchie';
|
||||
|
||||
console.log(`[StoreDiscovery] Starting discovery for platform: ${platform}`);
|
||||
|
||||
try {
|
||||
// Get states to discover
|
||||
const statesResult = await pool.query(`
|
||||
SELECT code FROM states WHERE active = true ORDER BY code
|
||||
`);
|
||||
const stateCodes = statesResult.rows.map(r => r.code);
|
||||
|
||||
if (stateCodes.length === 0) {
|
||||
return { success: true, storesDiscovered: 0, message: 'No active states to discover' };
|
||||
}
|
||||
|
||||
let totalDiscovered = 0;
|
||||
let totalPromoted = 0;
|
||||
|
||||
// Run discovery for each state
|
||||
const crawler = new DiscoveryCrawler(pool);
|
||||
|
||||
for (const stateCode of stateCodes) {
|
||||
// Heartbeat before each state
|
||||
await ctx.heartbeat();
|
||||
|
||||
console.log(`[StoreDiscovery] Discovering stores in ${stateCode}...`);
|
||||
|
||||
try {
|
||||
const result = await crawler.discoverState(stateCode);
|
||||
totalDiscovered += result.locationsDiscovered || 0;
|
||||
totalPromoted += result.locationsPromoted || 0;
|
||||
console.log(`[StoreDiscovery] ${stateCode}: discovered ${result.locationsDiscovered}, promoted ${result.locationsPromoted}`);
|
||||
} catch (error: any) {
|
||||
console.error(`[StoreDiscovery] Error discovering ${stateCode}:`, error.message);
|
||||
// Continue with other states
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`[StoreDiscovery] Complete: ${totalDiscovered} discovered, ${totalPromoted} promoted`);
|
||||
|
||||
return {
|
||||
success: true,
|
||||
storesDiscovered: totalDiscovered,
|
||||
storesPromoted: totalPromoted,
|
||||
statesProcessed: stateCodes.length,
|
||||
newStoreIds: [], // Would be populated with actual new store IDs for chaining
|
||||
};
|
||||
} catch (error: any) {
|
||||
console.error(`[StoreDiscovery] Error:`, error.message);
|
||||
return {
|
||||
success: false,
|
||||
error: error.message,
|
||||
};
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user