feat: Stealth worker system with mandatory proxy rotation

## Worker System
- Role-agnostic workers that can handle any task type
- Pod-based architecture with StatefulSet (5-15 pods, 5 workers each)
- Custom pod names (Aethelgard, Xylos, Kryll, etc.)
- Worker registry with friendly names and resource monitoring
- Hub-and-spoke visualization on JobQueue page

## Stealth & Anti-Detection (REQUIRED)
- Proxies are MANDATORY - workers fail to start without active proxies
- CrawlRotator initializes on worker startup
- Loads proxies from `proxies` table
- Auto-rotates proxy + fingerprint on 403 errors
- 12 browser fingerprints (Chrome, Firefox, Safari, Edge)
- Locale/timezone matching for geographic consistency

## Task System
- Renamed product_resync → product_refresh
- Task chaining: store_discovery → entry_point → product_discovery
- Priority-based claiming with FOR UPDATE SKIP LOCKED
- Heartbeat and stale task recovery

## UI Updates
- JobQueue: Pod visualization, resource monitoring on hover
- WorkersDashboard: Simplified worker list
- Removed unused filters from task list

## Other
- IP2Location service for visitor analytics
- Findagram consumer features scaffolding
- Documentation updates

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
Kelly
2025-12-10 00:44:59 -07:00
parent 0295637ed6
commit 56cc171287
61 changed files with 8591 additions and 2076 deletions

View File

@@ -0,0 +1,134 @@
/**
* IP2Location Service
*
* Uses local IP2Location LITE DB3 database for IP geolocation.
* No external API calls, no rate limits.
*
* Database: IP2Location LITE DB3 (free, monthly updates)
* Fields: country, region, city, latitude, longitude
*/
import path from 'path';
import fs from 'fs';
// @ts-ignore - no types for ip2location-nodejs
const { IP2Location } = require('ip2location-nodejs');
const DB_PATH = process.env.IP2LOCATION_DB_PATH ||
path.join(__dirname, '../../data/ip2location/IP2LOCATION-LITE-DB5.BIN');
let ip2location: any = null;
let dbLoaded = false;
/**
* Initialize IP2Location database
*/
export function initIP2Location(): boolean {
if (dbLoaded) return true;
try {
if (!fs.existsSync(DB_PATH)) {
console.warn(`IP2Location database not found at: ${DB_PATH}`);
console.warn('Run: ./scripts/download-ip2location.sh to download');
return false;
}
ip2location = new IP2Location();
ip2location.open(DB_PATH);
dbLoaded = true;
console.log('IP2Location database loaded successfully');
return true;
} catch (err) {
console.error('Failed to load IP2Location database:', err);
return false;
}
}
/**
* Close IP2Location database
*/
export function closeIP2Location(): void {
if (ip2location) {
ip2location.close();
ip2location = null;
dbLoaded = false;
}
}
export interface GeoLocation {
city: string | null;
state: string | null;
stateCode: string | null;
country: string | null;
countryCode: string | null;
lat: number | null;
lng: number | null;
}
/**
* Lookup IP address location
*
* @param ip - IPv4 or IPv6 address
* @returns Location data or null if not found
*/
export function lookupIP(ip: string): GeoLocation | null {
// Skip private/localhost IPs
if (!ip || ip === '127.0.0.1' || ip === '::1' ||
ip.startsWith('192.168.') || ip.startsWith('10.') ||
ip.startsWith('172.16.') || ip.startsWith('172.17.') ||
ip.startsWith('::ffff:127.') || ip.startsWith('::ffff:192.168.') ||
ip.startsWith('::ffff:10.')) {
return null;
}
// Strip IPv6 prefix if present
const cleanIP = ip.replace(/^::ffff:/, '');
// Initialize on first use if not already loaded
if (!dbLoaded) {
if (!initIP2Location()) {
return null;
}
}
try {
const result = ip2location.getAll(cleanIP);
if (!result || result.ip === '?' || result.countryShort === '-') {
return null;
}
// DB3 LITE doesn't include lat/lng - would need DB5+ for that
const lat = typeof result.latitude === 'number' && result.latitude !== 0 ? result.latitude : null;
const lng = typeof result.longitude === 'number' && result.longitude !== 0 ? result.longitude : null;
return {
city: result.city !== '-' ? result.city : null,
state: result.region !== '-' ? result.region : null,
stateCode: null, // DB3 doesn't include state codes
country: result.countryLong !== '-' ? result.countryLong : null,
countryCode: result.countryShort !== '-' ? result.countryShort : null,
lat,
lng,
};
} catch (err) {
console.error('IP2Location lookup error:', err);
return null;
}
}
/**
* Check if IP2Location database is available
*/
export function isIP2LocationAvailable(): boolean {
if (dbLoaded) return true;
return fs.existsSync(DB_PATH);
}
// Export singleton-style interface
export default {
init: initIP2Location,
close: closeIP2Location,
lookup: lookupIP,
isAvailable: isIP2LocationAvailable,
};