Creationshop/cannaiq
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(preflight): Use Evomi proxy API before falling back to DB pool

Browse Source 
The preflight was only checking DB proxy pool which was empty.
Now checks Evomi API first (when configured), then falls back to DB.

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
Kelly
2025-12-13 17:14:23 -07:00
parent 1490c60d2a
commit d779a08bbf
1 changed files with 26 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

35
backend/src/services/puppeteer-preflight.ts
View File

@@ -13,7 +13,7 @@
* Based on test-intercept.js which successfully captures 1000+ products * Based on test-intercept.js which successfully captures 1000+ products
*/ */
import { PreflightResult, CrawlRotator } from './crawl-rotator'; import { PreflightResult, CrawlRotator, getEvomiConfig, buildEvomiProxyUrl } from './crawl-rotator';
// GraphQL hash for FilteredProducts query - MUST match CLAUDE.md // GraphQL hash for FilteredProducts query - MUST match CLAUDE.md
const FILTERED_PRODUCTS_HASH = 'ee29c060826dc41c527e470e9ae502c9b2c169720faa0a9f5d25e1b9a530a4a0'; const FILTERED_PRODUCTS_HASH = 'ee29c060826dc41c527e470e9ae502c9b2c169720faa0a9f5d25e1b9a530a4a0';
@@ -105,27 +105,42 @@ export async function runPuppeteerPreflight(
let browser: any = null; let browser: any = null;
try { try {
// Step 0: Get a proxy from the pool // Step 0: Get a proxy - prefer Evomi API, fall back to DB pool
let proxyUrl: string | null = null; let proxyUrl: string | null = null;
let expectedProxyHost: string | null = null; let expectedProxyHost: string | null = null;
if (crawlRotator) { // Try Evomi first (dynamic residential proxies)
const evomiConfig = getEvomiConfig();
if (evomiConfig.enabled) {
// Use AZ as default state for preflight testing
const evomiProxy = buildEvomiProxyUrl('AZ', 'preflight-test');
if (evomiProxy) {
result.proxyAvailable = true;
proxyUrl = evomiProxy.url;
expectedProxyHost = evomiConfig.host;
result.expectedProxyIp = expectedProxyHost;
console.log(`[PuppeteerPreflight] Using Evomi proxy: ${evomiProxy.geo}`);
}
}
// Fall back to DB pool if Evomi not available
if (!proxyUrl && crawlRotator) {
const currentProxy = crawlRotator.proxy.getCurrent(); const currentProxy = crawlRotator.proxy.getCurrent();
if (currentProxy) { if (currentProxy) {
result.proxyAvailable = true; result.proxyAvailable = true;
proxyUrl = crawlRotator.proxy.getProxyUrl(currentProxy); proxyUrl = crawlRotator.proxy.getProxyUrl(currentProxy);
expectedProxyHost = currentProxy.host; expectedProxyHost = currentProxy.host;
result.expectedProxyIp = expectedProxyHost; result.expectedProxyIp = expectedProxyHost;
console.log(`[PuppeteerPreflight] Using proxy: ${currentProxy.host}:${currentProxy.port}`); console.log(`[PuppeteerPreflight] Using DB proxy: ${currentProxy.host}:${currentProxy.port}`);
} else { }
result.error = 'No proxy available from pool'; }
// No proxy available from either source
if (!proxyUrl) {
result.error = 'No proxy available (Evomi not configured, DB pool empty)';
console.log(`[PuppeteerPreflight] FAILED - No proxy available`); console.log(`[PuppeteerPreflight] FAILED - No proxy available`);
return result; return result;
} }
} else {
console.log(`[PuppeteerPreflight] WARNING: No CrawlRotator provided - using direct connection`);
result.proxyAvailable = true; // No proxy needed for direct
}
// Dynamic imports to avoid loading Puppeteer unless needed // Dynamic imports to avoid loading Puppeteer unless needed
const puppeteer = require('puppeteer-extra'); const puppeteer = require('puppeteer-extra');