56cc171287
## Worker System - Role-agnostic workers that can handle any task type - Pod-based architecture with StatefulSet (5-15 pods, 5 workers each) - Custom pod names (Aethelgard, Xylos, Kryll, etc.) - Worker registry with friendly names and resource monitoring - Hub-and-spoke visualization on JobQueue page ## Stealth & Anti-Detection (REQUIRED) - Proxies are MANDATORY - workers fail to start without active proxies - CrawlRotator initializes on worker startup - Loads proxies from `proxies` table - Auto-rotates proxy + fingerprint on 403 errors - 12 browser fingerprints (Chrome, Firefox, Safari, Edge) - Locale/timezone matching for geographic consistency ## Task System - Renamed product_resync → product_refresh - Task chaining: store_discovery → entry_point → product_discovery - Priority-based claiming with FOR UPDATE SKIP LOCKED - Heartbeat and stale task recovery ## UI Updates - JobQueue: Pod visualization, resource monitoring on hover - WorkersDashboard: Simplified worker list - Removed unused filters from task list ## Other - IP2Location service for visitor analytics - Findagram consumer features scaffolding - Documentation updates 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
94 lines
2.1 KiB
TypeScript
94 lines
2.1 KiB
TypeScript
#!/usr/bin/env npx tsx
|
|
/**
|
|
* Start Pod - Simulates a Kubernetes pod locally
|
|
*
|
|
* Starts 5 workers with a pod name from the predefined list.
|
|
*
|
|
* Usage:
|
|
* npx tsx src/tasks/start-pod.ts <pod-index>
|
|
* npx tsx src/tasks/start-pod.ts 0 # Starts pod "Aethelgard" with 5 workers
|
|
* npx tsx src/tasks/start-pod.ts 1 # Starts pod "Xylos" with 5 workers
|
|
*/
|
|
|
|
import { spawn } from 'child_process';
|
|
import path from 'path';
|
|
|
|
const POD_NAMES = [
|
|
'Aethelgard',
|
|
'Xylos',
|
|
'Kryll',
|
|
'Coriolis',
|
|
'Dimidium',
|
|
'Veridia',
|
|
'Zetani',
|
|
'Talos IV',
|
|
'Onyx',
|
|
'Celestia',
|
|
'Gormand',
|
|
'Betha',
|
|
'Ragnar',
|
|
'Syphon',
|
|
'Axiom',
|
|
'Nadir',
|
|
'Terra Nova',
|
|
'Acheron',
|
|
'Nexus',
|
|
'Vespera',
|
|
'Helios Prime',
|
|
'Oasis',
|
|
'Mordina',
|
|
'Cygnus',
|
|
'Umbra',
|
|
];
|
|
|
|
const WORKERS_PER_POD = 5;
|
|
|
|
async function main() {
|
|
const podIndex = parseInt(process.argv[2] ?? '0', 10);
|
|
|
|
if (podIndex < 0 || podIndex >= POD_NAMES.length) {
|
|
console.error(`Invalid pod index: ${podIndex}. Must be 0-${POD_NAMES.length - 1}`);
|
|
process.exit(1);
|
|
}
|
|
|
|
const podName = POD_NAMES[podIndex];
|
|
console.log(`[Pod] Starting pod "${podName}" with ${WORKERS_PER_POD} workers...`);
|
|
|
|
const workerScript = path.join(__dirname, 'task-worker.ts');
|
|
const workers: ReturnType<typeof spawn>[] = [];
|
|
|
|
for (let i = 1; i <= WORKERS_PER_POD; i++) {
|
|
const workerId = `${podName}-worker-${i}`;
|
|
|
|
const worker = spawn('npx', ['tsx', workerScript], {
|
|
env: {
|
|
...process.env,
|
|
WORKER_ID: workerId,
|
|
POD_NAME: podName,
|
|
},
|
|
stdio: 'inherit',
|
|
});
|
|
|
|
workers.push(worker);
|
|
console.log(`[Pod] Started worker ${i}/${WORKERS_PER_POD}: ${workerId}`);
|
|
}
|
|
|
|
// Handle shutdown
|
|
const shutdown = () => {
|
|
console.log(`\n[Pod] Shutting down pod "${podName}"...`);
|
|
workers.forEach(w => w.kill('SIGTERM'));
|
|
setTimeout(() => process.exit(0), 2000);
|
|
};
|
|
|
|
process.on('SIGTERM', shutdown);
|
|
process.on('SIGINT', shutdown);
|
|
|
|
// Keep the process alive
|
|
await new Promise(() => {});
|
|
}
|
|
|
|
main().catch(err => {
|
|
console.error('[Pod] Fatal error:', err);
|
|
process.exit(1);
|
|
});
|