feat(workers): Add proxy reload, staggered tasks, and bulk proxy import
- Periodic proxy reload: Workers now reload proxies every 60s to pick up changes - Staggered task scheduling: New API endpoints for creating tasks with delays - Bulk proxy import: Script supports multiple URL formats including host:port:user:pass - Proxy URL column: Migration 086 adds proxy_url for non-standard formats Key changes: - crawl-rotator.ts: Added reloadIfStale(), isStale(), setReloadInterval() - task-worker.ts: Calls reloadIfStale() in main loop - task-service.ts: Added createStaggeredTasks() and createAZStoreTasks() - tasks.ts: Added POST /batch/staggered and /batch/az-stores endpoints - import-proxies.ts: New script for bulk proxy import - CLAUDE.md: Documented staggered task workflow 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
@@ -976,6 +976,123 @@ router.post('/migration/full-migrate', async (req: Request, res: Response) => {
|
||||
}
|
||||
});
|
||||
|
||||
// ============================================================
|
||||
// STAGGERED BATCH TASK CREATION
|
||||
// ============================================================
|
||||
|
||||
/**
|
||||
* POST /api/tasks/batch/staggered
|
||||
* Create multiple tasks with staggered start times
|
||||
*
|
||||
* This endpoint prevents resource contention when creating many tasks by
|
||||
* staggering their scheduled_for timestamps. Each task becomes eligible
|
||||
* for claiming only after its scheduled time.
|
||||
*
|
||||
* WORKFLOW:
|
||||
* 1. Tasks created with scheduled_for = NOW() + (index * stagger_seconds)
|
||||
* 2. Worker claims task only when scheduled_for <= NOW()
|
||||
* 3. Worker runs preflight on EVERY task claim
|
||||
* 4. If preflight passes, worker executes task
|
||||
* 5. If preflight fails, task released back to pending for another worker
|
||||
*
|
||||
* Body:
|
||||
* - dispensary_ids: number[] (required) - Array of dispensary IDs
|
||||
* - role: TaskRole (required) - 'product_refresh' | 'product_discovery'
|
||||
* - stagger_seconds: number (default: 15) - Seconds between each task start
|
||||
* - platform: string (default: 'dutchie')
|
||||
* - method: 'curl' | 'http' | null (default: null)
|
||||
*/
|
||||
router.post('/batch/staggered', async (req: Request, res: Response) => {
|
||||
try {
|
||||
const {
|
||||
dispensary_ids,
|
||||
role,
|
||||
stagger_seconds = 15,
|
||||
platform = 'dutchie',
|
||||
method = null,
|
||||
} = req.body;
|
||||
|
||||
if (!dispensary_ids || !Array.isArray(dispensary_ids) || dispensary_ids.length === 0) {
|
||||
return res.status(400).json({ error: 'dispensary_ids array is required' });
|
||||
}
|
||||
|
||||
if (!role) {
|
||||
return res.status(400).json({ error: 'role is required' });
|
||||
}
|
||||
|
||||
const result = await taskService.createStaggeredTasks(
|
||||
dispensary_ids,
|
||||
role as TaskRole,
|
||||
stagger_seconds,
|
||||
platform,
|
||||
method
|
||||
);
|
||||
|
||||
const totalDuration = (dispensary_ids.length - 1) * stagger_seconds;
|
||||
const estimatedEndTime = new Date(Date.now() + totalDuration * 1000);
|
||||
|
||||
res.status(201).json({
|
||||
success: true,
|
||||
created: result.created,
|
||||
task_ids: result.taskIds,
|
||||
stagger_seconds,
|
||||
total_duration_seconds: totalDuration,
|
||||
estimated_completion: estimatedEndTime.toISOString(),
|
||||
message: `Created ${result.created} staggered ${role} tasks (${stagger_seconds}s apart, ~${Math.ceil(totalDuration / 60)} min total)`,
|
||||
});
|
||||
} catch (error: unknown) {
|
||||
console.error('Error creating staggered tasks:', error);
|
||||
res.status(500).json({ error: 'Failed to create staggered tasks' });
|
||||
}
|
||||
});
|
||||
|
||||
/**
|
||||
* POST /api/tasks/batch/az-stores
|
||||
* Convenience endpoint to create staggered tasks for Arizona stores
|
||||
*
|
||||
* Body:
|
||||
* - total_tasks: number (default: 24) - Total tasks to create
|
||||
* - stagger_seconds: number (default: 15) - Seconds between each task
|
||||
* - split_roles: boolean (default: true) - Split between product_refresh and product_discovery
|
||||
*/
|
||||
router.post('/batch/az-stores', async (req: Request, res: Response) => {
|
||||
try {
|
||||
const {
|
||||
total_tasks = 24,
|
||||
stagger_seconds = 15,
|
||||
split_roles = true,
|
||||
} = req.body;
|
||||
|
||||
const result = await taskService.createAZStoreTasks(
|
||||
total_tasks,
|
||||
stagger_seconds,
|
||||
split_roles
|
||||
);
|
||||
|
||||
const totalDuration = (result.total - 1) * stagger_seconds;
|
||||
const estimatedEndTime = new Date(Date.now() + totalDuration * 1000);
|
||||
|
||||
res.status(201).json({
|
||||
success: true,
|
||||
total: result.total,
|
||||
product_refresh: result.product_refresh,
|
||||
product_discovery: result.product_discovery,
|
||||
task_ids: result.taskIds,
|
||||
stagger_seconds,
|
||||
total_duration_seconds: totalDuration,
|
||||
estimated_completion: estimatedEndTime.toISOString(),
|
||||
message: `Created ${result.total} staggered tasks for AZ stores (${result.product_refresh} refresh, ${result.product_discovery} discovery)`,
|
||||
});
|
||||
} catch (error: unknown) {
|
||||
console.error('Error creating AZ store tasks:', error);
|
||||
res.status(500).json({ error: 'Failed to create AZ store tasks' });
|
||||
}
|
||||
});
|
||||
|
||||
// ============================================================
|
||||
// TASK POOL MANAGEMENT
|
||||
// ============================================================
|
||||
|
||||
/**
|
||||
* GET /api/tasks/pool/status
|
||||
* Check if task pool is paused
|
||||
|
||||
Reference in New Issue
Block a user