56cc171287
## Worker System - Role-agnostic workers that can handle any task type - Pod-based architecture with StatefulSet (5-15 pods, 5 workers each) - Custom pod names (Aethelgard, Xylos, Kryll, etc.) - Worker registry with friendly names and resource monitoring - Hub-and-spoke visualization on JobQueue page ## Stealth & Anti-Detection (REQUIRED) - Proxies are MANDATORY - workers fail to start without active proxies - CrawlRotator initializes on worker startup - Loads proxies from `proxies` table - Auto-rotates proxy + fingerprint on 403 errors - 12 browser fingerprints (Chrome, Firefox, Safari, Edge) - Locale/timezone matching for geographic consistency ## Task System - Renamed product_resync → product_refresh - Task chaining: store_discovery → entry_point → product_discovery - Priority-based claiming with FOR UPDATE SKIP LOCKED - Heartbeat and stale task recovery ## UI Updates - JobQueue: Pod visualization, resource monitoring on hover - WorkersDashboard: Simplified worker list - Removed unused filters from task list ## Other - IP2Location service for visitor analytics - Findagram consumer features scaffolding - Documentation updates 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
67 lines
2.4 KiB
JSON
Executable File
67 lines
2.4 KiB
JSON
Executable File
{
|
|
"name": "dutchie-menus-backend",
|
|
"version": "1.5.1",
|
|
"description": "Backend API for Dutchie Menus scraper and management",
|
|
"main": "dist/index.js",
|
|
"scripts": {
|
|
"dev": "tsx watch src/index.ts",
|
|
"dev:worker": "tsx watch src/cli.ts --worker",
|
|
"build": "tsc",
|
|
"start": "node dist/index.js",
|
|
"start:worker": "node dist/cli.js --worker",
|
|
"worker": "tsx src/cli.ts --worker",
|
|
"migrate": "tsx src/db/migrate.ts",
|
|
"seed": "tsx src/db/seed.ts",
|
|
"migrate:az": "tsx src/dutchie-az/db/migrate.ts",
|
|
"health:az": "tsx -e \"import { healthCheck } from './src/dutchie-az/db/connection'; (async()=>{ const ok=await healthCheck(); console.log(ok?'AZ DB healthy':'AZ DB NOT reachable'); process.exit(ok?0:1); })();\"",
|
|
"system:smoke-test": "tsx src/scripts/system-smoke-test.ts",
|
|
"discovery:dt:cities:auto": "tsx src/dutchie-az/discovery/discovery-dt-cities-auto.ts",
|
|
"discovery:dt:cities:manual": "tsx src/dutchie-az/discovery/discovery-dt-cities-manual-seed.ts",
|
|
"discovery:dt:locations": "tsx src/dutchie-az/discovery/discovery-dt-locations-from-cities.ts",
|
|
"backfill:legacy:canonical": "tsx src/scripts/backfill-legacy-to-canonical.ts",
|
|
"seed:dt:cities:bulk": "tsx src/scripts/seed-dt-cities-bulk.ts"
|
|
},
|
|
"dependencies": {
|
|
"@types/bcryptjs": "^3.0.0",
|
|
"axios": "^1.6.2",
|
|
"bcrypt": "^5.1.1",
|
|
"bcryptjs": "^3.0.3",
|
|
"bullmq": "^5.65.1",
|
|
"cheerio": "^1.1.2",
|
|
"cors": "^2.8.5",
|
|
"dotenv": "^16.3.1",
|
|
"express": "^4.18.2",
|
|
"express-rate-limit": "^7.1.5",
|
|
"helmet": "^7.1.0",
|
|
"https-proxy-agent": "^7.0.2",
|
|
"ioredis": "^5.8.2",
|
|
"ip2location-nodejs": "^9.7.0",
|
|
"ipaddr.js": "^2.2.0",
|
|
"jsonwebtoken": "^9.0.2",
|
|
"minio": "^7.1.3",
|
|
"node-cron": "^3.0.3",
|
|
"pg": "^8.11.3",
|
|
"playwright": "^1.56.1",
|
|
"playwright-extra": "^4.3.6",
|
|
"puppeteer": "^21.0.0",
|
|
"puppeteer-extra": "^3.3.6",
|
|
"puppeteer-extra-plugin-stealth": "^2.11.2",
|
|
"sharp": "^0.32.0",
|
|
"socks-proxy-agent": "^8.0.2",
|
|
"uuid": "^9.0.1",
|
|
"zod": "^3.22.4"
|
|
},
|
|
"devDependencies": {
|
|
"@types/bcrypt": "^5.0.2",
|
|
"@types/cors": "^2.8.17",
|
|
"@types/express": "^4.17.21",
|
|
"@types/jsonwebtoken": "^9.0.5",
|
|
"@types/node": "^20.10.5",
|
|
"@types/node-cron": "^3.0.11",
|
|
"@types/pg": "^8.15.6",
|
|
"@types/uuid": "^9.0.7",
|
|
"tsx": "^4.7.0",
|
|
"typescript": "^5.3.3"
|
|
}
|
|
}
|