cannaiq/backend/scripts/test-treez-stealth.ts

import puppeteer from 'puppeteer-extra';
import StealthPlugin from 'puppeteer-extra-plugin-stealth';
import fs from 'fs';

puppeteer.use(StealthPlugin());

async function sleep(ms: number): Promise<void> {
  return new Promise(resolve => setTimeout(resolve, ms));
}

async function main() {
  const browser = await puppeteer.launch({
    headless: 'new',
    args: ['--no-sandbox', '--disable-setuid-sandbox'],
  });

  const page = await browser.newPage();
  await page.setViewport({ width: 1920, height: 1080 });
  
  const capturedProducts: any[] = [];
  
  // Use CDP to intercept responses
  const client = await page.target().createCDPSession();
  await client.send('Network.enable');
  
  client.on('Network.responseReceived', async (event) => {
    const url = event.response.url;
    if (url.includes('gapcommerceapi.com/product/search') && event.response.status === 200) {
      try {
        const response = await client.send('Network.getResponseBody', {
          requestId: event.requestId,
        });
        
        const body = response.base64Encoded 
          ? Buffer.from(response.body, 'base64').toString('utf8')
          : response.body;
          
        const json = JSON.parse(body);
        const products = json.hits?.hits?.map((h: any) => h._source) || [];
        capturedProducts.push(...products);
        console.log('Captured ' + products.length + ' products (total: ' + capturedProducts.length + ')');
        
      } catch (err: any) {
        // Ignore errors
      }
    }
  });

  console.log('Loading page with Stealth plugin...\n');

  await page.goto('https://shop.bestdispensary.com/shop', {
    waitUntil: 'networkidle2',
    timeout: 60000
  });
  await sleep(3000);

  // Bypass age gate
  const ageGate = await page.$('[data-testid="age-gate-modal"]');
  if (ageGate) {
    console.log('Bypassing age gate...');
    const btn = await page.$('[data-testid="age-gate-submit-button"]');
    if (btn) await btn.click();
    await sleep(3000);
  }

  // Wait for API calls
  await sleep(5000);
  
  console.log('Initial capture: ' + capturedProducts.length + ' products');

  // Scroll and click load more
  console.log('\nScrolling and clicking Load More...');
  for (let i = 0; i < 30; i++) {
    await page.evaluate(() => window.scrollTo(0, document.body.scrollHeight));
    await sleep(1000);
    
    try {
      const btn = await page.$('button.collection__load-more');
      if (btn) {
        await btn.click();
        await sleep(2000);
      }
    } catch {}
    
    if (i % 5 === 0) {
      console.log('Progress: ' + capturedProducts.length + ' products');
    }
  }

  console.log('\n=== RESULTS ===\n');
  
  if (capturedProducts.length > 0) {
    const seen = new Set();
    const unique = capturedProducts.filter(p => {
      const id = p.id || p.productId;
      if (!id || seen.has(id)) return false;
      seen.add(id);
      return true;
    });
    
    console.log('Total captured: ' + capturedProducts.length);
    console.log('Unique products: ' + unique.length);
    console.log('\nFields available:');
    console.log(Object.keys(unique[0]).sort().join('\n'));
    console.log('\nSample product:\n' + JSON.stringify(unique[0], null, 2));
    
    fs.writeFileSync('/tmp/treez-products.json', JSON.stringify(unique, null, 2));
    console.log('\nSaved to /tmp/treez-products.json');
  } else {
    console.log('No products captured - API still blocking');
  }

  await browser.close();
}

main();