Creationshop/cannaiq
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(worker): Use Evomi API first, DB proxies as fallback

Browse Source 
- Check Evomi API availability before waiting for DB proxies
- If EVOMI_USER/EVOMI_PASS configured, proceed immediately
- Only fall back to DB proxy polling if Evomi not configured
- Added clear comments explaining proxy initialization order

This fixes workers getting stuck waiting for DB proxies when
Evomi API is available for on-demand geo-targeted proxies.

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
Kelly
2025-12-13 16:45:52 -07:00
parent 3d0ea21007
commit cf99ef9e09
3 changed files with 220 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
backend/src/platforms/treez/client.ts
View File

@@ -298,9 +298,10 @@ export async function bypassAgeGate(page: Page): Promise<boolean> {
/**
* Build menu URL for a store
* Uses /brands page which contains all products (not just homepage carousels)
*/
export function buildMenuUrl(storeId: string, customerType: 'ADULT' | 'MEDICAL' = 'ADULT'): string {
return `https://${storeId}.treez.io/onlinemenu/?customerType=${customerType}`;
return `https://${storeId}.treez.io/onlinemenu/brands?customerType=${customerType}`;
}
/**

39
backend/src/tasks/task-worker.ts
View File

@@ -533,24 +533,53 @@ export class TaskWorker {
}
try {
// ============================================================
// PROXY INITIALIZATION ORDER:
// 1. Check Evomi API first (dynamic residential proxies)
// 2. Fall back to DB proxies if Evomi not configured
//
// Evomi provides geo-targeted proxies on-demand via API.
// DB proxies are static/datacenter proxies as fallback.
// ============================================================
// Import Evomi config checker
const { getEvomiConfig } = await import('../services/crawl-rotator');
const evomiConfig = getEvomiConfig();
if (evomiConfig.enabled) {
// Evomi API is configured - we can get proxies on-demand
// No need to wait for DB proxies
console.log(`[TaskWorker] Evomi API configured (${evomiConfig.host}:${evomiConfig.port}) - proxies available on-demand`);
// Still initialize rotator for user-agent rotation
await this.crawlRotator.initialize();
setCrawlRotator(this.crawlRotator);
console.log(`[TaskWorker] Stealth initialized: ${this.crawlRotator.userAgent.getCount()} fingerprints, Evomi API for proxies`);
return;
}
// Evomi not configured - fall back to DB proxies
console.log(`[TaskWorker] Evomi API not configured, falling back to DB proxies...`);
while (attempts < maxAttempts) {
try {
// Load proxies from database
// Load proxies from database (fallback)
await this.crawlRotator.initialize();
const stats = this.crawlRotator.proxy.getStats();
if (stats.activeProxies > 0) {
console.log(`[TaskWorker] Loaded ${stats.activeProxies} proxies (${stats.avgSuccessRate.toFixed(1)}% avg success rate)`);
console.log(`[TaskWorker] Loaded ${stats.activeProxies} DB proxies (${stats.avgSuccessRate.toFixed(1)}% avg success rate)`);
// Wire rotator to Dutchie client - proxies will be used for ALL requests
setCrawlRotator(this.crawlRotator);
console.log(`[TaskWorker] Stealth initialized: ${this.crawlRotator.userAgent.getCount()} fingerprints, proxy REQUIRED for all requests`);
console.log(`[TaskWorker] Stealth initialized: ${this.crawlRotator.userAgent.getCount()} fingerprints, DB proxies`);
return;
}
attempts++;
console.log(`[TaskWorker] No active proxies available (attempt ${attempts}). Waiting for proxies...`);
console.log(`[TaskWorker] No DB proxies available (attempt ${attempts}). Waiting...`);
// Wait for either notification or timeout
await new Promise<void>((resolve) => {
@@ -564,7 +593,7 @@ export class TaskWorker {
}
}
throw new Error(`No active proxies available after waiting ${MAX_WAIT_MINUTES} minutes. Add proxies to the database.`);
throw new Error(`No proxies available after ${MAX_WAIT_MINUTES} minutes. Configure EVOMI_USER/EVOMI_PASS or add proxies to database.`);
} finally {
// Clean up LISTEN connection
if (notifyClient) {