## Worker System - Role-agnostic workers that can handle any task type - Pod-based architecture with StatefulSet (5-15 pods, 5 workers each) - Custom pod names (Aethelgard, Xylos, Kryll, etc.) - Worker registry with friendly names and resource monitoring - Hub-and-spoke visualization on JobQueue page ## Stealth & Anti-Detection (REQUIRED) - Proxies are MANDATORY - workers fail to start without active proxies - CrawlRotator initializes on worker startup - Loads proxies from `proxies` table - Auto-rotates proxy + fingerprint on 403 errors - 12 browser fingerprints (Chrome, Firefox, Safari, Edge) - Locale/timezone matching for geographic consistency ## Task System - Renamed product_resync → product_refresh - Task chaining: store_discovery → entry_point → product_discovery - Priority-based claiming with FOR UPDATE SKIP LOCKED - Heartbeat and stale task recovery ## UI Updates - JobQueue: Pod visualization, resource monitoring on hover - WorkersDashboard: Simplified worker list - Removed unused filters from task list ## Other - IP2Location service for visitor analytics - Findagram consumer features scaffolding - Documentation updates 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
135 lines
3.5 KiB
TypeScript
135 lines
3.5 KiB
TypeScript
/**
|
|
* IP2Location Service
|
|
*
|
|
* Uses local IP2Location LITE DB3 database for IP geolocation.
|
|
* No external API calls, no rate limits.
|
|
*
|
|
* Database: IP2Location LITE DB3 (free, monthly updates)
|
|
* Fields: country, region, city, latitude, longitude
|
|
*/
|
|
|
|
import path from 'path';
|
|
import fs from 'fs';
|
|
|
|
// @ts-ignore - no types for ip2location-nodejs
|
|
const { IP2Location } = require('ip2location-nodejs');
|
|
|
|
const DB_PATH = process.env.IP2LOCATION_DB_PATH ||
|
|
path.join(__dirname, '../../data/ip2location/IP2LOCATION-LITE-DB5.BIN');
|
|
|
|
let ip2location: any = null;
|
|
let dbLoaded = false;
|
|
|
|
/**
|
|
* Initialize IP2Location database
|
|
*/
|
|
export function initIP2Location(): boolean {
|
|
if (dbLoaded) return true;
|
|
|
|
try {
|
|
if (!fs.existsSync(DB_PATH)) {
|
|
console.warn(`IP2Location database not found at: ${DB_PATH}`);
|
|
console.warn('Run: ./scripts/download-ip2location.sh to download');
|
|
return false;
|
|
}
|
|
|
|
ip2location = new IP2Location();
|
|
ip2location.open(DB_PATH);
|
|
dbLoaded = true;
|
|
console.log('IP2Location database loaded successfully');
|
|
return true;
|
|
} catch (err) {
|
|
console.error('Failed to load IP2Location database:', err);
|
|
return false;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Close IP2Location database
|
|
*/
|
|
export function closeIP2Location(): void {
|
|
if (ip2location) {
|
|
ip2location.close();
|
|
ip2location = null;
|
|
dbLoaded = false;
|
|
}
|
|
}
|
|
|
|
export interface GeoLocation {
|
|
city: string | null;
|
|
state: string | null;
|
|
stateCode: string | null;
|
|
country: string | null;
|
|
countryCode: string | null;
|
|
lat: number | null;
|
|
lng: number | null;
|
|
}
|
|
|
|
/**
|
|
* Lookup IP address location
|
|
*
|
|
* @param ip - IPv4 or IPv6 address
|
|
* @returns Location data or null if not found
|
|
*/
|
|
export function lookupIP(ip: string): GeoLocation | null {
|
|
// Skip private/localhost IPs
|
|
if (!ip || ip === '127.0.0.1' || ip === '::1' ||
|
|
ip.startsWith('192.168.') || ip.startsWith('10.') ||
|
|
ip.startsWith('172.16.') || ip.startsWith('172.17.') ||
|
|
ip.startsWith('::ffff:127.') || ip.startsWith('::ffff:192.168.') ||
|
|
ip.startsWith('::ffff:10.')) {
|
|
return null;
|
|
}
|
|
|
|
// Strip IPv6 prefix if present
|
|
const cleanIP = ip.replace(/^::ffff:/, '');
|
|
|
|
// Initialize on first use if not already loaded
|
|
if (!dbLoaded) {
|
|
if (!initIP2Location()) {
|
|
return null;
|
|
}
|
|
}
|
|
|
|
try {
|
|
const result = ip2location.getAll(cleanIP);
|
|
|
|
if (!result || result.ip === '?' || result.countryShort === '-') {
|
|
return null;
|
|
}
|
|
|
|
// DB3 LITE doesn't include lat/lng - would need DB5+ for that
|
|
const lat = typeof result.latitude === 'number' && result.latitude !== 0 ? result.latitude : null;
|
|
const lng = typeof result.longitude === 'number' && result.longitude !== 0 ? result.longitude : null;
|
|
|
|
return {
|
|
city: result.city !== '-' ? result.city : null,
|
|
state: result.region !== '-' ? result.region : null,
|
|
stateCode: null, // DB3 doesn't include state codes
|
|
country: result.countryLong !== '-' ? result.countryLong : null,
|
|
countryCode: result.countryShort !== '-' ? result.countryShort : null,
|
|
lat,
|
|
lng,
|
|
};
|
|
} catch (err) {
|
|
console.error('IP2Location lookup error:', err);
|
|
return null;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Check if IP2Location database is available
|
|
*/
|
|
export function isIP2LocationAvailable(): boolean {
|
|
if (dbLoaded) return true;
|
|
return fs.existsSync(DB_PATH);
|
|
}
|
|
|
|
// Export singleton-style interface
|
|
export default {
|
|
init: initIP2Location,
|
|
close: closeIP2Location,
|
|
lookup: lookupIP,
|
|
isAvailable: isIP2LocationAvailable,
|
|
};
|