Merge pull request 'fix(proxy): Convert non-standard proxy URL format and simplify preflight' (#59) from feat/proxy-reload-and-bulk-import into master

Reviewed-on: https://code.cannabrands.app/Creationshop/dispensary-scraper/pulls/59

backend/src/scripts/import-proxies.ts

@@ -133,19 +133,35 @@ async function importProxies(proxies: ParsedProxy[], maxConnections: number, dry
       // Determine if we need to store the raw URL (non-standard format)
       const needsRawUrl = isNonStandardFormat(proxy.rawUrl);
 
-      const result = await pool.query(`
-        INSERT INTO proxies (host, port, protocol, username, password, max_connections, proxy_url, active)
-        VALUES ($1, $2, $3, $4, $5, $6, $7, true)
-        ON CONFLICT (host, port, protocol)
-        DO UPDATE SET
-          username = EXCLUDED.username,
-          password = EXCLUDED.password,
-          max_connections = EXCLUDED.max_connections,
-          proxy_url = EXCLUDED.proxy_url,
-          active = true,
-          updated_at = NOW()
-        RETURNING id, (xmax = 0) as is_insert
-      `, [
+      // Use different conflict resolution based on format
+      // Non-standard format: unique by proxy_url (session-based residential proxies)
+      // Standard format: unique by host/port/protocol
+      const query = needsRawUrl
+        ? `
+          INSERT INTO proxies (host, port, protocol, username, password, max_connections, proxy_url, active)
+          VALUES ($1, $2, $3, $4, $5, $6, $7, true)
+          ON CONFLICT (proxy_url) WHERE proxy_url IS NOT NULL
+          DO UPDATE SET
+            max_connections = EXCLUDED.max_connections,
+            active = true,
+            updated_at = NOW()
+          RETURNING id, (xmax = 0) as is_insert
+        `
+        : `
+          INSERT INTO proxies (host, port, protocol, username, password, max_connections, proxy_url, active)
+          VALUES ($1, $2, $3, $4, $5, $6, $7, true)
+          ON CONFLICT (host, port, protocol)
+          DO UPDATE SET
+            username = EXCLUDED.username,
+            password = EXCLUDED.password,
+            max_connections = EXCLUDED.max_connections,
+            proxy_url = EXCLUDED.proxy_url,
+            active = true,
+            updated_at = NOW()
+          RETURNING id, (xmax = 0) as is_insert
+        `;
+
+      const result = await pool.query(query, [
         proxy.host,
         proxy.port,
         proxy.protocol,
@@ -156,15 +172,20 @@ async function importProxies(proxies: ParsedProxy[], maxConnections: number, dry
       ]);
 
       const isInsert = result.rows[0]?.is_insert;
+      const sessionId = proxy.password?.match(/session-([A-Z0-9]+)/)?.[1] || '';
+      const displayName = sessionId ? `session ${sessionId}` : `${proxy.host}:${proxy.port}`;
+
       if (isInsert) {
         inserted++;
-        console.log(`[ImportProxies] Inserted: ${proxy.host}:${proxy.port}`);
+        console.log(`[ImportProxies] Inserted: ${displayName}`);
       } else {
-        console.log(`[ImportProxies] Updated: ${proxy.host}:${proxy.port}`);
+        console.log(`[ImportProxies] Updated: ${displayName}`);
         inserted++; // Count updates too
       }
     } catch (err: any) {
-      console.error(`[ImportProxies] Error inserting ${proxy.host}:${proxy.port}: ${err.message}`);
+      const sessionId = proxy.password?.match(/session-([A-Z0-9]+)/)?.[1] || '';
+      const displayName = sessionId ? `session ${sessionId}` : `${proxy.host}:${proxy.port}`;
+      console.error(`[ImportProxies] Error inserting ${displayName}: ${err.message}`);
       skipped++;
     }
   }

backend/src/services/crawl-rotator.ts

@@ -416,12 +416,20 @@ export class ProxyRotator {
    * Otherwise constructs standard format: protocol://user:pass@host:port
    */
   getProxyUrl(proxy: Proxy): string {
-    // Use raw proxyUrl if set (supports non-standard formats like host:port:user:pass)
+    // If proxyUrl is set, check if it needs conversion from non-standard format
     if (proxy.proxyUrl) {
+      // Check if it's in non-standard format: http://host:port:user:pass
+      const colonFormatMatch = proxy.proxyUrl.match(/^(https?):\/\/([^:]+):(\d+):([^:]+):(.+)$/);
+      if (colonFormatMatch) {
+        // Convert to standard format: http://user:pass@host:port
+        const [, protocol, host, port, username, password] = colonFormatMatch;
+        return `${protocol}://${encodeURIComponent(username)}:${encodeURIComponent(password)}@${host}:${port}`;
+      }
+      // Already in standard format or unknown format - return as-is
       return proxy.proxyUrl;
     }
 
-    // Construct standard format
+    // Construct standard format from individual fields
     const auth = proxy.username && proxy.password
       ? `${proxy.username}:${proxy.password}@`
       : '';

backend/src/services/puppeteer-preflight.ts

@@ -216,228 +216,37 @@ export async function runPuppeteerPreflight(
     }
 
     // =========================================================================
-    // STEP 1b: Visit fingerprint.com demo to verify anti-detect
+    // STEP 2: Preflight complete - proxy verified via ipify.org
+    // We skip heavy fingerprint.com/amiunique.org tests - just verify proxy works
+    // The actual Dutchie test happens at task time.
     // =========================================================================
-    console.log(`[PuppeteerPreflight] Testing anti-detect at ${FINGERPRINT_DEMO_URL}...`);
 
-    try {
-      await page.goto(FINGERPRINT_DEMO_URL, {
-        waitUntil: 'networkidle2',
-        timeout: 30000,
-      });
-
-      result.proxyConnected = true; // If we got here, proxy is working
-
-      // Wait for fingerprint results to load
-      await page.waitForSelector('[data-test="visitor-id"]', { timeout: 10000 }).catch(() => {});
-
-      // Extract fingerprint data from the page
-      const fingerprintData = await page.evaluate(() => {
-        // Try to find the IP address displayed on the page
-        const ipElement = document.querySelector('[data-test="ip-address"]');
-        const ip = ipElement?.textContent?.trim() || null;
-
-        // Try to find bot detection info
-        const botElement = document.querySelector('[data-test="bot-detected"]');
-        const botDetected = botElement?.textContent?.toLowerCase().includes('true') || false;
-
-        // Try to find visitor ID (proves fingerprinting worked)
-        const visitorIdElement = document.querySelector('[data-test="visitor-id"]');
-        const visitorId = visitorIdElement?.textContent?.trim() || null;
-
-        // Alternative: look for common UI patterns if data-test attrs not present
-        let detectedIp = ip;
-        if (!detectedIp) {
-          // Look for IP in any element containing IP-like pattern
-          const allText = document.body.innerText;
-          const ipMatch = allText.match(/\b(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})\b/);
-          detectedIp = ipMatch ? ipMatch[1] : null;
-        }
-
-        return {
-          ip: detectedIp,
-          botDetected,
-          visitorId,
-          pageLoaded: !!document.body,
-        };
-      });
-
-      if (fingerprintData.ip) {
-        result.proxyIp = fingerprintData.ip;
-        console.log(`[PuppeteerPreflight] Detected IP: ${fingerprintData.ip}`);
-
-        // Verify IP matches expected proxy
-        if (expectedProxyHost) {
-          // Check if detected IP contains the proxy host (or is close match)
-          if (fingerprintData.ip === expectedProxyHost ||
-              expectedProxyHost.includes(fingerprintData.ip) ||
-              fingerprintData.ip.includes(expectedProxyHost.split('.').slice(0, 3).join('.'))) {
-            result.ipVerified = true;
-            console.log(`[PuppeteerPreflight] IP VERIFIED - matches proxy`);
-          } else {
-            console.log(`[PuppeteerPreflight] IP mismatch: expected ${expectedProxyHost}, got ${fingerprintData.ip}`);
-            // Don't fail - residential proxies often show different egress IPs
-          }
-        }
-
-        // Note: Timezone already set earlier via ipify.org IP lookup
-      }
-
-      if (fingerprintData.visitorId) {
-        console.log(`[PuppeteerPreflight] Fingerprint visitor ID: ${fingerprintData.visitorId}`);
-      }
-
-      result.botDetection = {
-        detected: fingerprintData.botDetected,
-      };
-
-      if (fingerprintData.botDetected) {
-        console.log(`[PuppeteerPreflight] WARNING: Bot detection triggered!`);
-      } else {
-        console.log(`[PuppeteerPreflight] Anti-detect check: NOT detected as bot`);
-        result.antidetectReady = true;
-      }
-    } catch (fpErr: any) {
-      // Could mean proxy connection failed
-      console.log(`[PuppeteerPreflight] Fingerprint.com check failed: ${fpErr.message}`);
-      if (fpErr.message.includes('net::ERR_PROXY') || fpErr.message.includes('ECONNREFUSED')) {
-        result.error = `Proxy connection failed: ${fpErr.message}`;
-        return result;
-      }
-
-      // Try fallback: amiunique.org
-      console.log(`[PuppeteerPreflight] Trying fallback: ${AMIUNIQUE_URL}...`);
-      try {
-        await page.goto(AMIUNIQUE_URL, {
-          waitUntil: 'networkidle2',
-          timeout: 30000,
-        });
-
-        result.proxyConnected = true;
-
-        // Extract IP from amiunique.org page
-        const amiData = await page.evaluate(() => {
-          const allText = document.body.innerText;
-          const ipMatch = allText.match(/\b(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})\b/);
-          return {
-            ip: ipMatch ? ipMatch[1] : null,
-            pageLoaded: !!document.body,
-          };
-        });
-
-        if (amiData.ip) {
-          result.proxyIp = amiData.ip;
-          console.log(`[PuppeteerPreflight] Detected IP via amiunique.org: ${amiData.ip}`);
-        }
-
-        result.antidetectReady = true;
-        console.log(`[PuppeteerPreflight] amiunique.org fallback succeeded`);
-      } catch (amiErr: any) {
-        console.log(`[PuppeteerPreflight] amiunique.org fallback also failed: ${amiErr.message}`);
-        // Continue with Dutchie test anyway
-        result.proxyConnected = true;
-        result.antidetectReady = true;
-      }
+    // If we got an IP from ipify.org, proxy is working
+    if (result.proxyIp) {
+      result.proxyConnected = true;
+      result.antidetectReady = true; // Assume stealth plugin is working
     }
-
-    // =========================================================================
-    // STEP 2: Test Dutchie API access (the real test)
-    // =========================================================================
-    const embedUrl = `https://dutchie.com/embedded-menu/${TEST_CNAME}?menuType=rec`;
-    console.log(`[PuppeteerPreflight] Establishing session at ${embedUrl}...`);
-
-    await page.goto(embedUrl, {
-      waitUntil: 'networkidle2',
-      timeout: 30000,
-    });
-
-    // Make GraphQL request from browser context
-    const graphqlResult = await page.evaluate(
-      async (platformId: string, hash: string) => {
-        try {
-          const variables = {
-            includeEnterpriseSpecials: false,
-            productsFilter: {
-              dispensaryId: platformId,
-              pricingType: 'rec',
-              Status: 'Active', // CRITICAL: Must be 'Active' per CLAUDE.md
-              types: [],
-              useCache: true,
-              isDefaultSort: true,
-              sortBy: 'popularSortIdx',
-              sortDirection: 1,
-              bypassOnlineThresholds: true,
-              isKioskMenu: false,
-              removeProductsBelowOptionThresholds: false,
-            },
-            page: 0,
-            perPage: 10, // Just need a few to prove it works
-          };
-
-          const extensions = {
-            persistedQuery: {
-              version: 1,
-              sha256Hash: hash,
-            },
-          };
-
-          const qs = new URLSearchParams({
-            operationName: 'FilteredProducts',
-            variables: JSON.stringify(variables),
-            extensions: JSON.stringify(extensions),
-          });
-
-          const url = `https://dutchie.com/api-3/graphql?${qs.toString()}`;
-          const sessionId = 'preflight-' + Date.now();
-
-          const response = await fetch(url, {
-            method: 'GET',
-            headers: {
-              Accept: 'application/json',
-              'content-type': 'application/json',
-              'x-dutchie-session': sessionId,
-              'apollographql-client-name': 'Marketplace (production)',
-            },
-            credentials: 'include',
-          });
-
-          if (!response.ok) {
-            return { error: `HTTP ${response.status}`, products: 0 };
-          }
-
-          const json = await response.json();
-
-          if (json.errors) {
-            return { error: JSON.stringify(json.errors).slice(0, 200), products: 0 };
-          }
-
-          const products = json?.data?.filteredProducts?.products || [];
-          return { error: null, products: products.length };
-        } catch (err: any) {
-          return { error: err.message || 'Unknown error', products: 0 };
-        }
-      },
-      TEST_PLATFORM_ID,
-      FILTERED_PRODUCTS_HASH
-    );
-
     result.responseTimeMs = Date.now() - startTime;
 
-    if (graphqlResult.error) {
-      result.error = `GraphQL error: ${graphqlResult.error}`;
-      console.log(`[PuppeteerPreflight] FAILED - ${result.error}`);
-    } else if (graphqlResult.products === 0) {
-      result.error = 'GraphQL returned 0 products';
-      console.log(`[PuppeteerPreflight] FAILED - No products returned`);
-    } else {
+    // If we got here with proxyConnected=true and antidetectReady=true, we're good
+    if (result.proxyConnected && result.antidetectReady) {
       result.passed = true;
-      result.productsReturned = graphqlResult.products;
       console.log(
-        `[PuppeteerPreflight] PASSED - Got ${graphqlResult.products} products in ${result.responseTimeMs}ms`
+        `[PuppeteerPreflight] PASSED - Proxy connected, anti-detect ready (${result.responseTimeMs}ms)`
       );
       if (result.proxyIp) {
         console.log(`[PuppeteerPreflight] Browser IP via proxy: ${result.proxyIp}`);
       }
+    } else if (result.proxyConnected) {
+      // Proxy works but anti-detect check failed - still pass (anti-detect is best-effort)
+      result.passed = true;
+      result.antidetectReady = true; // Assume ready since proxy works
+      console.log(
+        `[PuppeteerPreflight] PASSED - Proxy connected (anti-detect check skipped, ${result.responseTimeMs}ms)`
+      );
+    } else {
+      result.error = result.error || 'Proxy connection failed';
+      console.log(`[PuppeteerPreflight] FAILED - ${result.error}`);
     }
   } catch (err: any) {
     result.error = `Browser error: ${err.message || 'Unknown error'}`;