feat: Stealth worker system with mandatory proxy rotation
## Worker System - Role-agnostic workers that can handle any task type - Pod-based architecture with StatefulSet (5-15 pods, 5 workers each) - Custom pod names (Aethelgard, Xylos, Kryll, etc.) - Worker registry with friendly names and resource monitoring - Hub-and-spoke visualization on JobQueue page ## Stealth & Anti-Detection (REQUIRED) - Proxies are MANDATORY - workers fail to start without active proxies - CrawlRotator initializes on worker startup - Loads proxies from `proxies` table - Auto-rotates proxy + fingerprint on 403 errors - 12 browser fingerprints (Chrome, Firefox, Safari, Edge) - Locale/timezone matching for geographic consistency ## Task System - Renamed product_resync → product_refresh - Task chaining: store_discovery → entry_point → product_discovery - Priority-based claiming with FOR UPDATE SKIP LOCKED - Heartbeat and stale task recovery ## UI Updates - JobQueue: Pod visualization, resource monitoring on hover - WorkersDashboard: Simplified worker list - Removed unused filters from task list ## Other - IP2Location service for visitor analytics - Findagram consumer features scaffolding - Documentation updates 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
134
backend/src/services/ip2location.ts
Normal file
134
backend/src/services/ip2location.ts
Normal file
@@ -0,0 +1,134 @@
|
||||
/**
|
||||
* IP2Location Service
|
||||
*
|
||||
* Uses local IP2Location LITE DB3 database for IP geolocation.
|
||||
* No external API calls, no rate limits.
|
||||
*
|
||||
* Database: IP2Location LITE DB3 (free, monthly updates)
|
||||
* Fields: country, region, city, latitude, longitude
|
||||
*/
|
||||
|
||||
import path from 'path';
|
||||
import fs from 'fs';
|
||||
|
||||
// @ts-ignore - no types for ip2location-nodejs
|
||||
const { IP2Location } = require('ip2location-nodejs');
|
||||
|
||||
const DB_PATH = process.env.IP2LOCATION_DB_PATH ||
|
||||
path.join(__dirname, '../../data/ip2location/IP2LOCATION-LITE-DB5.BIN');
|
||||
|
||||
let ip2location: any = null;
|
||||
let dbLoaded = false;
|
||||
|
||||
/**
|
||||
* Initialize IP2Location database
|
||||
*/
|
||||
export function initIP2Location(): boolean {
|
||||
if (dbLoaded) return true;
|
||||
|
||||
try {
|
||||
if (!fs.existsSync(DB_PATH)) {
|
||||
console.warn(`IP2Location database not found at: ${DB_PATH}`);
|
||||
console.warn('Run: ./scripts/download-ip2location.sh to download');
|
||||
return false;
|
||||
}
|
||||
|
||||
ip2location = new IP2Location();
|
||||
ip2location.open(DB_PATH);
|
||||
dbLoaded = true;
|
||||
console.log('IP2Location database loaded successfully');
|
||||
return true;
|
||||
} catch (err) {
|
||||
console.error('Failed to load IP2Location database:', err);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Close IP2Location database
|
||||
*/
|
||||
export function closeIP2Location(): void {
|
||||
if (ip2location) {
|
||||
ip2location.close();
|
||||
ip2location = null;
|
||||
dbLoaded = false;
|
||||
}
|
||||
}
|
||||
|
||||
export interface GeoLocation {
|
||||
city: string | null;
|
||||
state: string | null;
|
||||
stateCode: string | null;
|
||||
country: string | null;
|
||||
countryCode: string | null;
|
||||
lat: number | null;
|
||||
lng: number | null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Lookup IP address location
|
||||
*
|
||||
* @param ip - IPv4 or IPv6 address
|
||||
* @returns Location data or null if not found
|
||||
*/
|
||||
export function lookupIP(ip: string): GeoLocation | null {
|
||||
// Skip private/localhost IPs
|
||||
if (!ip || ip === '127.0.0.1' || ip === '::1' ||
|
||||
ip.startsWith('192.168.') || ip.startsWith('10.') ||
|
||||
ip.startsWith('172.16.') || ip.startsWith('172.17.') ||
|
||||
ip.startsWith('::ffff:127.') || ip.startsWith('::ffff:192.168.') ||
|
||||
ip.startsWith('::ffff:10.')) {
|
||||
return null;
|
||||
}
|
||||
|
||||
// Strip IPv6 prefix if present
|
||||
const cleanIP = ip.replace(/^::ffff:/, '');
|
||||
|
||||
// Initialize on first use if not already loaded
|
||||
if (!dbLoaded) {
|
||||
if (!initIP2Location()) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
const result = ip2location.getAll(cleanIP);
|
||||
|
||||
if (!result || result.ip === '?' || result.countryShort === '-') {
|
||||
return null;
|
||||
}
|
||||
|
||||
// DB3 LITE doesn't include lat/lng - would need DB5+ for that
|
||||
const lat = typeof result.latitude === 'number' && result.latitude !== 0 ? result.latitude : null;
|
||||
const lng = typeof result.longitude === 'number' && result.longitude !== 0 ? result.longitude : null;
|
||||
|
||||
return {
|
||||
city: result.city !== '-' ? result.city : null,
|
||||
state: result.region !== '-' ? result.region : null,
|
||||
stateCode: null, // DB3 doesn't include state codes
|
||||
country: result.countryLong !== '-' ? result.countryLong : null,
|
||||
countryCode: result.countryShort !== '-' ? result.countryShort : null,
|
||||
lat,
|
||||
lng,
|
||||
};
|
||||
} catch (err) {
|
||||
console.error('IP2Location lookup error:', err);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if IP2Location database is available
|
||||
*/
|
||||
export function isIP2LocationAvailable(): boolean {
|
||||
if (dbLoaded) return true;
|
||||
return fs.existsSync(DB_PATH);
|
||||
}
|
||||
|
||||
// Export singleton-style interface
|
||||
export default {
|
||||
init: initIP2Location,
|
||||
close: closeIP2Location,
|
||||
lookup: lookupIP,
|
||||
isAvailable: isIP2LocationAvailable,
|
||||
};
|
||||
Reference in New Issue
Block a user