feat(workers): Add dual-transport preflight system

Workers now run both curl and http (Puppeteer) preflights on startup:
- curl-preflight.ts: Tests axios + proxy via httpbin.org
- puppeteer-preflight.ts: Tests browser + StealthPlugin via fingerprint.com
  (with amiunique.org fallback)
- Migration 084: Adds preflight columns to worker_registry and method
  column to worker_tasks
- Workers report preflight status, IP, fingerprint, and response time
- Tasks can require specific transport method (curl/http)
- Dashboard shows Transport column with preflight status badges

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
Kelly
2025-12-11 22:47:52 -07:00
parent a35976b9e9
commit cdab71a1ee
7 changed files with 1085 additions and 8 deletions

View File

@@ -51,6 +51,7 @@ export interface WorkerTask {
platform: string | null;
status: TaskStatus;
priority: number;
method: 'curl' | 'http' | null; // Transport method: curl=axios/proxy, http=Puppeteer/browser
scheduled_for: Date | null;
worker_id: string | null;
claimed_at: Date | null;
@@ -152,23 +153,33 @@ class TaskService {
* Claim a task atomically for a worker
* If role is null, claims ANY available task (role-agnostic worker)
* Returns null if task pool is paused.
*
* @param role - Task role to claim, or null for any task
* @param workerId - Worker ID claiming the task
* @param curlPassed - Whether worker passed curl preflight (default true for backward compat)
* @param httpPassed - Whether worker passed http/Puppeteer preflight (default false)
*/
async claimTask(role: TaskRole | null, workerId: string): Promise<WorkerTask | null> {
async claimTask(
role: TaskRole | null,
workerId: string,
curlPassed: boolean = true,
httpPassed: boolean = false
): Promise<WorkerTask | null> {
// Check if task pool is paused - don't claim any tasks
if (isTaskPoolPaused()) {
return null;
}
if (role) {
// Role-specific claiming - use the SQL function
// Role-specific claiming - use the SQL function with preflight capabilities
const result = await pool.query(
`SELECT * FROM claim_task($1, $2)`,
[role, workerId]
`SELECT * FROM claim_task($1, $2, $3, $4)`,
[role, workerId, curlPassed, httpPassed]
);
return (result.rows[0] as WorkerTask) || null;
}
// Role-agnostic claiming - claim ANY pending task
// Role-agnostic claiming - claim ANY pending task matching worker capabilities
const result = await pool.query(`
UPDATE worker_tasks
SET
@@ -179,6 +190,12 @@ class TaskService {
SELECT id FROM worker_tasks
WHERE status = 'pending'
AND (scheduled_for IS NULL OR scheduled_for <= NOW())
-- Method compatibility: worker must have passed the required preflight
AND (
method IS NULL -- No preference, any worker can claim
OR (method = 'curl' AND $2 = TRUE)
OR (method = 'http' AND $3 = TRUE)
)
-- Exclude stores that already have an active task
AND (dispensary_id IS NULL OR dispensary_id NOT IN (
SELECT dispensary_id FROM worker_tasks
@@ -190,7 +207,7 @@ class TaskService {
FOR UPDATE SKIP LOCKED
)
RETURNING *
`, [workerId]);
`, [workerId, curlPassed, httpPassed]);
return (result.rows[0] as WorkerTask) || null;
}