cannaiq/backend/src/services/puppeteer-preflight.ts

/**
 * Puppeteer Preflight - Verify browser-based transport works with anti-detect
 *
 * Uses Puppeteer + StealthPlugin to:
 * 1. Launch headless browser with stealth mode + PROXY
 * 2. Visit fingerprint.com demo to verify anti-detect and confirm proxy IP
 * 3. Establish session by visiting Dutchie embedded menu
 * 4. Make GraphQL request from browser context
 * 5. Verify we get a valid response (not blocked)
 *
 * Use case: Anti-detect scraping that needs real browser fingerprint through proxy
 *
 * Based on test-intercept.js which successfully captures 1000+ products
 */

import { PreflightResult, CrawlRotator } from './crawl-rotator';

// GraphQL hash for FilteredProducts query - MUST match CLAUDE.md
const FILTERED_PRODUCTS_HASH = 'ee29c060826dc41c527e470e9ae502c9b2c169720faa0a9f5d25e1b9a530a4a0';

// Test dispensary - AZ-Deeply-Rooted (known working)
const TEST_CNAME = 'AZ-Deeply-Rooted';
const TEST_PLATFORM_ID = '6405ef617056e8014d79101b';

// Anti-detect verification sites (primary + fallback)
const FINGERPRINT_DEMO_URL = 'https://demo.fingerprint.com/';
const AMIUNIQUE_URL = 'https://amiunique.org/fingerprint';

export interface PuppeteerPreflightResult extends PreflightResult {
  method: 'http';
  /** Number of products returned (proves API access) */
  productsReturned?: number;
  /** Browser user agent used */
  browserUserAgent?: string;
  /** Bot detection result from fingerprint.com */
  botDetection?: {
    detected: boolean;
    probability?: number;
    type?: string;
  };
  /** Expected proxy IP (from pool) */
  expectedProxyIp?: string;
  /** Whether IP verification passed (detected IP matches proxy) */
  ipVerified?: boolean;
}

/**
 * Run Puppeteer preflight check with proxy
 * Tests browser-based access with anti-detect verification via fingerprint.com
 *
 * @param crawlRotator - CrawlRotator instance to get proxy from pool
 */
export async function runPuppeteerPreflight(
  crawlRotator?: CrawlRotator
): Promise<PuppeteerPreflightResult> {
  const result: PuppeteerPreflightResult = {
    method: 'http',
    passed: false,
    proxyAvailable: false,
    proxyConnected: false,
    antidetectReady: false,
    proxyIp: null,
    fingerprint: null,
    error: null,
    responseTimeMs: null,
    productsReturned: 0,
    ipVerified: false,
  };

  let browser: any = null;

  try {
    // Step 0: Get a proxy from the pool
    let proxyUrl: string | null = null;
    let expectedProxyHost: string | null = null;

    if (crawlRotator) {
      const currentProxy = crawlRotator.proxy.getCurrent();
      if (currentProxy) {
        result.proxyAvailable = true;
        proxyUrl = crawlRotator.proxy.getProxyUrl(currentProxy);
        expectedProxyHost = currentProxy.host;
        result.expectedProxyIp = expectedProxyHost;
        console.log(`[PuppeteerPreflight] Using proxy: ${currentProxy.host}:${currentProxy.port}`);
      } else {
        result.error = 'No proxy available from pool';
        console.log(`[PuppeteerPreflight] FAILED - No proxy available`);
        return result;
      }
    } else {
      console.log(`[PuppeteerPreflight] WARNING: No CrawlRotator provided - using direct connection`);
      result.proxyAvailable = true; // No proxy needed for direct
    }

    // Dynamic imports to avoid loading Puppeteer unless needed
    const puppeteer = require('puppeteer-extra');
    const StealthPlugin = require('puppeteer-extra-plugin-stealth');
    puppeteer.use(StealthPlugin());

    const startTime = Date.now();

    // Build browser args
    const browserArgs = ['--no-sandbox', '--disable-setuid-sandbox'];
    if (proxyUrl) {
      // Extract host:port for Puppeteer (it handles auth separately)
      const proxyUrlParsed = new URL(proxyUrl);
      browserArgs.push(`--proxy-server=${proxyUrlParsed.host}`);
    }

    // Launch browser with stealth + proxy
    browser = await puppeteer.launch({
      headless: 'new',
      args: browserArgs,
    });

    const page = await browser.newPage();

    // If proxy has auth, set it up
    if (proxyUrl) {
      const proxyUrlParsed = new URL(proxyUrl);
      if (proxyUrlParsed.username && proxyUrlParsed.password) {
        await page.authenticate({
          username: decodeURIComponent(proxyUrlParsed.username),
          password: decodeURIComponent(proxyUrlParsed.password),
        });
      }
    }

    // Get browser user agent
    const userAgent = await page.evaluate(() => navigator.userAgent);
    result.browserUserAgent = userAgent;
    result.fingerprint = {
      userAgent,
      browserName: 'Chrome (Puppeteer)',
      deviceCategory: 'desktop',
    };

    // =========================================================================
    // STEP 1: Visit fingerprint.com demo to verify anti-detect and get IP
    // =========================================================================
    console.log(`[PuppeteerPreflight] Testing anti-detect at ${FINGERPRINT_DEMO_URL}...`);

    try {
      await page.goto(FINGERPRINT_DEMO_URL, {
        waitUntil: 'networkidle2',
        timeout: 30000,
      });

      result.proxyConnected = true; // If we got here, proxy is working

      // Wait for fingerprint results to load
      await page.waitForSelector('[data-test="visitor-id"]', { timeout: 10000 }).catch(() => {});

      // Extract fingerprint data from the page
      const fingerprintData = await page.evaluate(() => {
        // Try to find the IP address displayed on the page
        const ipElement = document.querySelector('[data-test="ip-address"]');
        const ip = ipElement?.textContent?.trim() || null;

        // Try to find bot detection info
        const botElement = document.querySelector('[data-test="bot-detected"]');
        const botDetected = botElement?.textContent?.toLowerCase().includes('true') || false;

        // Try to find visitor ID (proves fingerprinting worked)
        const visitorIdElement = document.querySelector('[data-test="visitor-id"]');
        const visitorId = visitorIdElement?.textContent?.trim() || null;

        // Alternative: look for common UI patterns if data-test attrs not present
        let detectedIp = ip;
        if (!detectedIp) {
          // Look for IP in any element containing IP-like pattern
          const allText = document.body.innerText;
          const ipMatch = allText.match(/\b(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})\b/);
          detectedIp = ipMatch ? ipMatch[1] : null;
        }

        return {
          ip: detectedIp,
          botDetected,
          visitorId,
          pageLoaded: !!document.body,
        };
      });

      if (fingerprintData.ip) {
        result.proxyIp = fingerprintData.ip;
        console.log(`[PuppeteerPreflight] Detected IP: ${fingerprintData.ip}`);

        // Verify IP matches expected proxy
        if (expectedProxyHost) {
          // Check if detected IP contains the proxy host (or is close match)
          if (fingerprintData.ip === expectedProxyHost ||
              expectedProxyHost.includes(fingerprintData.ip) ||
              fingerprintData.ip.includes(expectedProxyHost.split('.').slice(0, 3).join('.'))) {
            result.ipVerified = true;
            console.log(`[PuppeteerPreflight] IP VERIFIED - matches proxy`);
          } else {
            console.log(`[PuppeteerPreflight] IP mismatch: expected ${expectedProxyHost}, got ${fingerprintData.ip}`);
            // Don't fail - residential proxies often show different egress IPs
          }
        }
      }

      if (fingerprintData.visitorId) {
        console.log(`[PuppeteerPreflight] Fingerprint visitor ID: ${fingerprintData.visitorId}`);
      }

      result.botDetection = {
        detected: fingerprintData.botDetected,
      };

      if (fingerprintData.botDetected) {
        console.log(`[PuppeteerPreflight] WARNING: Bot detection triggered!`);
      } else {
        console.log(`[PuppeteerPreflight] Anti-detect check: NOT detected as bot`);
        result.antidetectReady = true;
      }
    } catch (fpErr: any) {
      // Could mean proxy connection failed
      console.log(`[PuppeteerPreflight] Fingerprint.com check failed: ${fpErr.message}`);
      if (fpErr.message.includes('net::ERR_PROXY') || fpErr.message.includes('ECONNREFUSED')) {
        result.error = `Proxy connection failed: ${fpErr.message}`;
        return result;
      }

      // Try fallback: amiunique.org
      console.log(`[PuppeteerPreflight] Trying fallback: ${AMIUNIQUE_URL}...`);
      try {
        await page.goto(AMIUNIQUE_URL, {
          waitUntil: 'networkidle2',
          timeout: 30000,
        });

        result.proxyConnected = true;

        // Extract IP from amiunique.org page
        const amiData = await page.evaluate(() => {
          const allText = document.body.innerText;
          const ipMatch = allText.match(/\b(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})\b/);
          return {
            ip: ipMatch ? ipMatch[1] : null,
            pageLoaded: !!document.body,
          };
        });

        if (amiData.ip) {
          result.proxyIp = amiData.ip;
          console.log(`[PuppeteerPreflight] Detected IP via amiunique.org: ${amiData.ip}`);
        }

        result.antidetectReady = true;
        console.log(`[PuppeteerPreflight] amiunique.org fallback succeeded`);
      } catch (amiErr: any) {
        console.log(`[PuppeteerPreflight] amiunique.org fallback also failed: ${amiErr.message}`);
        // Continue with Dutchie test anyway
        result.proxyConnected = true;
        result.antidetectReady = true;
      }
    }

    // =========================================================================
    // STEP 2: Test Dutchie API access (the real test)
    // =========================================================================
    const embedUrl = `https://dutchie.com/embedded-menu/${TEST_CNAME}?menuType=rec`;
    console.log(`[PuppeteerPreflight] Establishing session at ${embedUrl}...`);

    await page.goto(embedUrl, {
      waitUntil: 'networkidle2',
      timeout: 30000,
    });

    // Make GraphQL request from browser context
    const graphqlResult = await page.evaluate(
      async (platformId: string, hash: string) => {
        try {
          const variables = {
            includeEnterpriseSpecials: false,
            productsFilter: {
              dispensaryId: platformId,
              pricingType: 'rec',
              Status: 'Active', // CRITICAL: Must be 'Active' per CLAUDE.md
              types: [],
              useCache: true,
              isDefaultSort: true,
              sortBy: 'popularSortIdx',
              sortDirection: 1,
              bypassOnlineThresholds: true,
              isKioskMenu: false,
              removeProductsBelowOptionThresholds: false,
            },
            page: 0,
            perPage: 10, // Just need a few to prove it works
          };

          const extensions = {
            persistedQuery: {
              version: 1,
              sha256Hash: hash,
            },
          };

          const qs = new URLSearchParams({
            operationName: 'FilteredProducts',
            variables: JSON.stringify(variables),
            extensions: JSON.stringify(extensions),
          });

          const url = `https://dutchie.com/api-3/graphql?${qs.toString()}`;
          const sessionId = 'preflight-' + Date.now();

          const response = await fetch(url, {
            method: 'GET',
            headers: {
              Accept: 'application/json',
              'content-type': 'application/json',
              'x-dutchie-session': sessionId,
              'apollographql-client-name': 'Marketplace (production)',
            },
            credentials: 'include',
          });

          if (!response.ok) {
            return { error: `HTTP ${response.status}`, products: 0 };
          }

          const json = await response.json();

          if (json.errors) {
            return { error: JSON.stringify(json.errors).slice(0, 200), products: 0 };
          }

          const products = json?.data?.filteredProducts?.products || [];
          return { error: null, products: products.length };
        } catch (err: any) {
          return { error: err.message || 'Unknown error', products: 0 };
        }
      },
      TEST_PLATFORM_ID,
      FILTERED_PRODUCTS_HASH
    );

    result.responseTimeMs = Date.now() - startTime;

    if (graphqlResult.error) {
      result.error = `GraphQL error: ${graphqlResult.error}`;
      console.log(`[PuppeteerPreflight] FAILED - ${result.error}`);
    } else if (graphqlResult.products === 0) {
      result.error = 'GraphQL returned 0 products';
      console.log(`[PuppeteerPreflight] FAILED - No products returned`);
    } else {
      result.passed = true;
      result.productsReturned = graphqlResult.products;
      console.log(
        `[PuppeteerPreflight] PASSED - Got ${graphqlResult.products} products in ${result.responseTimeMs}ms`
      );
      if (result.proxyIp) {
        console.log(`[PuppeteerPreflight] Browser IP via proxy: ${result.proxyIp}`);
      }
    }
  } catch (err: any) {
    result.error = `Browser error: ${err.message || 'Unknown error'}`;
    console.log(`[PuppeteerPreflight] FAILED - ${result.error}`);
  } finally {
    if (browser) {
      await browser.close().catch(() => {});
    }
  }

  return result;
}

/**
 * Run Puppeteer preflight with retry
 * Retries once on failure to handle transient issues
 *
 * @param crawlRotator - CrawlRotator instance to get proxy from pool
 * @param maxRetries - Number of retry attempts (default 1)
 */
export async function runPuppeteerPreflightWithRetry(
  crawlRotator?: CrawlRotator,
  maxRetries: number = 1
): Promise<PuppeteerPreflightResult> {
  let lastResult: PuppeteerPreflightResult | null = null;

  for (let attempt = 0; attempt <= maxRetries; attempt++) {
    if (attempt > 0) {
      console.log(`[PuppeteerPreflight] Retry attempt ${attempt}/${maxRetries}...`);
      await new Promise((r) => setTimeout(r, 5000)); // Wait 5s between retries
    }

    lastResult = await runPuppeteerPreflight(crawlRotator);

    if (lastResult.passed) {
      return lastResult;
    }
  }

  return lastResult!;
}