cannaiq/backend/scripts/test-treez-scroll.ts

/**
 * Test aggressive scrolling to load all products
 */

import puppeteer from 'puppeteer-extra';
import StealthPlugin from 'puppeteer-extra-plugin-stealth';

puppeteer.use(StealthPlugin());

function sleep(ms: number): Promise<void> {
  return new Promise(r => setTimeout(r, ms));
}

async function main() {
  const browser = await puppeteer.launch({
    headless: 'new',
    args: ['--no-sandbox', '--disable-setuid-sandbox'],
  });

  const page = await browser.newPage();
  await page.setViewport({ width: 1920, height: 1080 });

  const capturedProducts: any[] = [];

  // CDP interception
  const client = await page.target().createCDPSession();
  await client.send('Network.enable');

  client.on('Network.responseReceived', async (event: any) => {
    if (event.response.url.includes('gapcommerceapi.com/product/search') && event.response.status === 200) {
      try {
        const response = await client.send('Network.getResponseBody', { requestId: event.requestId });
        const body = response.base64Encoded ? Buffer.from(response.body, 'base64').toString('utf8') : response.body;
        const json = JSON.parse(body);
        const products = json.hits?.hits?.map((h: any) => h._source) || [];
        capturedProducts.push(...products);
        console.log('Captured ' + products.length + ' (total: ' + capturedProducts.length + ')');
      } catch {}
    }
  });

  // Try direct treez.io URL - may have more products
  const url = process.argv[2] || 'https://best.treez.io/onlinemenu/';
  console.log('Loading ' + url);

  try {
    await page.goto(url, { waitUntil: 'networkidle0', timeout: 60000 });
  } catch (e: any) {
    console.log('Navigation warning: ' + e.message);
  }

  await sleep(5000);
  console.log('Current URL: ' + page.url());

  // Age gate
  const ageGate = await page.$('[data-testid="age-gate-modal"]');
  if (ageGate) {
    console.log('Bypassing age gate...');
    const btn = await page.$('[data-testid="age-gate-submit-button"]');
    if (btn) await btn.click();
    await sleep(3000);
  }

  console.log('After initial load: ' + capturedProducts.length + ' products');

  // Aggressive scrolling and clicking
  let lastCount = 0;
  let staleCount = 0;

  for (let i = 0; i < 60; i++) {
    await page.evaluate(() => window.scrollTo(0, document.body.scrollHeight));
    await sleep(800);

    try {
      const btn = await page.$('button.collection__load-more');
      if (btn) {
        const visible = await page.evaluate((b: Element) => {
          const rect = b.getBoundingClientRect();
          return rect.width > 0 && rect.height > 0;
        }, btn);

        if (visible) {
          await page.evaluate((b: Element) => b.scrollIntoView({ block: 'center' }), btn);
          await sleep(300);
          await btn.click();
          await sleep(2000);
          console.log('Clicked Load More at scroll ' + (i+1) + ' - ' + capturedProducts.length + ' products');
        }
      }
    } catch {}

    // Check for stale data
    if (capturedProducts.length === lastCount) {
      staleCount++;
      if (staleCount >= 5) {
        console.log('No new products for 5 iterations, stopping');
        break;
      }
    } else {
      staleCount = 0;
    }
    lastCount = capturedProducts.length;
  }

  console.log('\nFinal count: ' + capturedProducts.length + ' products');

  // Dedupe
  const seen = new Set<string>();
  const unique = capturedProducts.filter(p => {
    if (!p.id || seen.has(p.id)) return false;
    seen.add(p.id);
    return true;
  });

  console.log('Unique: ' + unique.length);

  // Categories
  const cats: Record<string, number> = {};
  unique.forEach(p => {
    cats[p.category] = (cats[p.category] || 0) + 1;
  });
  console.log('\nCategories:');
  Object.entries(cats).sort((a, b) => b[1] - a[1]).forEach(([c, n]) => console.log('  ' + c + ': ' + n));

  // Sample cannabis product
  const cannabis = unique.find(p => p.category === 'FLOWER' || p.category === 'VAPE');
  if (cannabis) {
    console.log('\nSample cannabis product:');
    console.log(JSON.stringify({
      id: cannabis.id,
      name: cannabis.name,
      brand: cannabis.brand,
      category: cannabis.category,
      subtype: cannabis.subtype,
      availableUnits: cannabis.availableUnits,
      customMinPrice: cannabis.customMinPrice,
    }, null, 2));
  }

  await browser.close();
}

main();