feat(tasks): Consolidate schedule management into task_schedules
- Add schedule CRUD endpoints to /api/tasks/schedules - Add Schedules section to TasksDashboard with edit/delete/bulk actions - Deprecate job_schedules table (entries disabled in DB) - Mark CrawlSchedulePage as deprecated (removed from menu) - Add deprecation comments to legacy schedule methods in api.ts - Add migration comments to workers.ts explaining consolidation Key changes: - Schedule management now at /admin/tasks instead of /admin/schedule - task_schedules uses interval_hours (simpler than base_interval_minutes + jitter) - All schedule routes placed before /:id to avoid Express route conflicts 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
Kelly
@@ -11,10 +11,17 @@
|
||||
* - Workers report heartbeats to worker_registry
|
||||
* - Workers are ROLE-AGNOSTIC by default (can handle any task type)
|
||||
*
|
||||
* Stealth & Anti-Detection:
|
||||
* PROXIES ARE REQUIRED - workers will fail to start if no proxies available.
|
||||
* Stealth & Anti-Detection (LAZY INITIALIZATION):
|
||||
* Workers start IMMEDIATELY without waiting for proxies.
|
||||
* Stealth systems (proxies, fingerprints, preflights) are initialized
|
||||
* on first task claim, not at worker startup.
|
||||
*
|
||||
* On startup, workers initialize the CrawlRotator which provides:
|
||||
* This allows workers to:
|
||||
* - Register and send heartbeats immediately
|
||||
* - Wait in main loop without blocking on proxy availability
|
||||
* - Initialize proxies/preflights only when tasks are actually available
|
||||
*
|
||||
* On first task claim attempt, workers initialize the CrawlRotator which provides:
|
||||
* - Proxy rotation: Loads proxies from `proxies` table, ALL requests use proxy
|
||||
* - User-Agent rotation: Cycles through realistic browser fingerprints
|
||||
* - Fingerprint rotation: Changes browser profile on blocks
|
||||
@@ -34,11 +41,16 @@
|
||||
*
|
||||
* Environment:
|
||||
* WORKER_ROLE - Which task role to process (optional, null = any task)
|
||||
* WORKER_ID - Optional custom worker ID (auto-generated if not provided)
|
||||
* POD_NAME - Kubernetes pod name (optional)
|
||||
* POD_NAME - K8s StatefulSet pod name (PRIMARY - use this for persistent identity)
|
||||
* WORKER_ID - Custom worker ID (fallback if POD_NAME not set)
|
||||
* POLL_INTERVAL_MS - How often to check for tasks (default: 5000)
|
||||
* HEARTBEAT_INTERVAL_MS - How often to update heartbeat (default: 30000)
|
||||
* API_BASE_URL - Backend API URL for registration (default: http://localhost:3010)
|
||||
*
|
||||
* Worker Identity:
|
||||
* Workers use POD_NAME as their worker_id for persistent identity across restarts.
|
||||
* In K8s StatefulSet, POD_NAME = "scraper-worker-0" through "scraper-worker-7".
|
||||
* This ensures workers re-register with the same ID instead of creating new entries.
|
||||
*/
|
||||
|
||||
import { Pool } from 'pg';
|
||||
@@ -209,6 +221,16 @@ export class TaskWorker {
|
||||
private preflightCurlResult: CurlPreflightResult | null = null;
|
||||
private preflightHttpResult: PuppeteerPreflightResult | null = null;
|
||||
|
||||
// ==========================================================================
|
||||
// LAZY INITIALIZATION FLAGS
|
||||
// ==========================================================================
|
||||
// Stealth/proxy initialization is deferred until first task claim.
|
||||
// Workers register immediately and enter main loop without blocking.
|
||||
// ==========================================================================
|
||||
private stealthInitialized: boolean = false;
|
||||
private preflightsCompleted: boolean = false;
|
||||
private initializingPromise: Promise<void> | null = null;
|
||||
|
||||
constructor(role: TaskRole | null = null, workerId?: string) {
|
||||
this.pool = getPool();
|
||||
this.role = role;
|
||||
@@ -293,9 +315,9 @@ export class TaskWorker {
|
||||
|
||||
/**
|
||||
* Initialize stealth systems (proxy rotation, fingerprints)
|
||||
* Called once on worker startup before processing any tasks.
|
||||
* Called LAZILY on first task claim attempt (NOT at worker startup).
|
||||
*
|
||||
* IMPORTANT: Proxies are REQUIRED. Workers will wait until proxies are available.
|
||||
* IMPORTANT: Proxies are REQUIRED to claim tasks. This method waits until proxies are available.
|
||||
* Workers listen for PostgreSQL NOTIFY 'proxy_added' to wake up immediately when proxies are added.
|
||||
*/
|
||||
private async initializeStealth(): Promise<void> {
|
||||
@@ -482,6 +504,51 @@ export class TaskWorker {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Lazy initialization of stealth systems.
|
||||
* Called BEFORE claiming first task (not at worker startup).
|
||||
* This allows workers to register and enter main loop immediately.
|
||||
*
|
||||
* Returns true if initialization succeeded, false otherwise.
|
||||
*/
|
||||
private async ensureStealthInitialized(): Promise<boolean> {
|
||||
// Already initialized
|
||||
if (this.stealthInitialized && this.preflightsCompleted) {
|
||||
return true;
|
||||
}
|
||||
|
||||
// Already initializing (prevent concurrent init attempts)
|
||||
if (this.initializingPromise) {
|
||||
await this.initializingPromise;
|
||||
return this.stealthInitialized && this.preflightsCompleted;
|
||||
}
|
||||
|
||||
console.log(`[TaskWorker] ${this.friendlyName} lazy-initializing stealth systems (first task claim)...`);
|
||||
|
||||
this.initializingPromise = (async () => {
|
||||
try {
|
||||
// Initialize proxy/fingerprint rotation
|
||||
await this.initializeStealth();
|
||||
this.stealthInitialized = true;
|
||||
|
||||
// Run dual-transport preflights
|
||||
await this.runDualPreflights();
|
||||
this.preflightsCompleted = true;
|
||||
|
||||
const preflightMsg = `curl=${this.preflightCurlPassed ? '✓' : '✗'} http=${this.preflightHttpPassed ? '✓' : '✗'}`;
|
||||
console.log(`[TaskWorker] ${this.friendlyName} stealth ready (${preflightMsg})`);
|
||||
} catch (err: any) {
|
||||
console.error(`[TaskWorker] ${this.friendlyName} stealth init failed: ${err.message}`);
|
||||
this.stealthInitialized = false;
|
||||
this.preflightsCompleted = false;
|
||||
}
|
||||
})();
|
||||
|
||||
await this.initializingPromise;
|
||||
this.initializingPromise = null;
|
||||
return this.stealthInitialized && this.preflightsCompleted;
|
||||
}
|
||||
|
||||
/**
|
||||
* Register worker with the registry (get friendly name)
|
||||
*/
|
||||
@@ -615,25 +682,22 @@ export class TaskWorker {
|
||||
|
||||
/**
|
||||
* Start the worker loop
|
||||
*
|
||||
* Workers start IMMEDIATELY without blocking on proxy/preflight init.
|
||||
* Stealth systems are lazy-initialized on first task claim.
|
||||
* This allows workers to register and send heartbeats even when proxies aren't ready.
|
||||
*/
|
||||
async start(): Promise<void> {
|
||||
this.isRunning = true;
|
||||
|
||||
// Initialize stealth systems (proxy rotation, fingerprints)
|
||||
await this.initializeStealth();
|
||||
|
||||
// Register with the API to get a friendly name
|
||||
// Register with the API to get a friendly name (non-blocking)
|
||||
await this.register();
|
||||
|
||||
// Run dual-transport preflights
|
||||
await this.runDualPreflights();
|
||||
|
||||
// Start registry heartbeat
|
||||
// Start registry heartbeat immediately
|
||||
this.startRegistryHeartbeat();
|
||||
|
||||
const roleMsg = this.role ? `for role: ${this.role}` : '(role-agnostic - any task)';
|
||||
const preflightMsg = `curl=${this.preflightCurlPassed ? '✓' : '✗'} http=${this.preflightHttpPassed ? '✓' : '✗'}`;
|
||||
console.log(`[TaskWorker] ${this.friendlyName} starting ${roleMsg} (${preflightMsg}, max ${this.maxConcurrentTasks} concurrent tasks)`);
|
||||
console.log(`[TaskWorker] ${this.friendlyName} starting ${roleMsg} (stealth=lazy, max ${this.maxConcurrentTasks} concurrent tasks)`);
|
||||
|
||||
while (this.isRunning) {
|
||||
try {
|
||||
@@ -687,6 +751,20 @@ export class TaskWorker {
|
||||
|
||||
// Try to claim more tasks if we have capacity
|
||||
if (this.canAcceptMoreTasks()) {
|
||||
// =================================================================
|
||||
// LAZY INITIALIZATION - Initialize stealth on first task claim
|
||||
// Workers start immediately and init proxies only when needed
|
||||
// =================================================================
|
||||
if (!this.stealthInitialized) {
|
||||
const initSuccess = await this.ensureStealthInitialized();
|
||||
if (!initSuccess) {
|
||||
// Init failed - wait and retry next loop
|
||||
console.log(`[TaskWorker] ${this.friendlyName} stealth init failed, waiting before retry...`);
|
||||
await this.sleep(30000);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
// Pass preflight capabilities to only claim compatible tasks
|
||||
const task = await taskService.claimTask(
|
||||
this.role,
|
||||
@@ -922,7 +1000,10 @@ async function main(): Promise<void> {
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const workerId = process.env.WORKER_ID;
|
||||
// Use POD_NAME for persistent identity in K8s StatefulSet
|
||||
// This ensures workers keep the same ID across restarts
|
||||
// Falls back to WORKER_ID, then generates UUID if neither is set
|
||||
const workerId = process.env.POD_NAME || process.env.WORKER_ID;
|
||||
// Pass null for role-agnostic, or the specific role
|
||||
const worker = new TaskWorker(role || null, workerId);
|
||||
|
||||
|
||||
Reference in New Issue
Block a user