feat(tasks): Add unified task-based worker architecture

Replace fragmented job systems (job_schedules, dispensary_crawl_jobs, SyncOrchestrator)
with a single unified task queue:

- Add worker_tasks table with atomic task claiming via SELECT FOR UPDATE SKIP LOCKED
- Add TaskService for CRUD, claiming, and capacity metrics
- Add TaskWorker with role-based handlers (resync, discovery, analytics)
- Add /api/tasks endpoints for management and migration from legacy systems
- Add TasksDashboard UI and integrate task counts into main dashboard
- Add comprehensive documentation

Task roles: store_discovery, entry_point_discovery, product_discovery, product_resync, analytics_refresh

Run workers with: WORKER_ROLE=product_resync npx tsx src/tasks/task-worker.ts

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
Kelly
2025-12-09 16:27:03 -07:00
parent 7f9cf559cf
commit 89c262ee20
18 changed files with 3167 additions and 2 deletions

View File

@@ -0,0 +1,67 @@
/**
* Store Discovery Handler
*
* Discovers new stores on a platform (e.g., Dutchie) by crawling
* location APIs and adding them to dutchie_discovery_locations.
*/
import { TaskContext, TaskResult } from '../task-worker';
import { DiscoveryCrawler } from '../../discovery/discovery-crawler';
export async function handleStoreDiscovery(ctx: TaskContext): Promise<TaskResult> {
const { pool, task } = ctx;
const platform = task.platform || 'dutchie';
console.log(`[StoreDiscovery] Starting discovery for platform: ${platform}`);
try {
// Get states to discover
const statesResult = await pool.query(`
SELECT code FROM states WHERE active = true ORDER BY code
`);
const stateCodes = statesResult.rows.map(r => r.code);
if (stateCodes.length === 0) {
return { success: true, storesDiscovered: 0, message: 'No active states to discover' };
}
let totalDiscovered = 0;
let totalPromoted = 0;
// Run discovery for each state
const crawler = new DiscoveryCrawler(pool);
for (const stateCode of stateCodes) {
// Heartbeat before each state
await ctx.heartbeat();
console.log(`[StoreDiscovery] Discovering stores in ${stateCode}...`);
try {
const result = await crawler.discoverState(stateCode);
totalDiscovered += result.locationsDiscovered || 0;
totalPromoted += result.locationsPromoted || 0;
console.log(`[StoreDiscovery] ${stateCode}: discovered ${result.locationsDiscovered}, promoted ${result.locationsPromoted}`);
} catch (error: any) {
console.error(`[StoreDiscovery] Error discovering ${stateCode}:`, error.message);
// Continue with other states
}
}
console.log(`[StoreDiscovery] Complete: ${totalDiscovered} discovered, ${totalPromoted} promoted`);
return {
success: true,
storesDiscovered: totalDiscovered,
storesPromoted: totalPromoted,
statesProcessed: stateCodes.length,
newStoreIds: [], // Would be populated with actual new store IDs for chaining
};
} catch (error: any) {
console.error(`[StoreDiscovery] Error:`, error.message);
return {
success: false,
error: error.message,
};
}
}