import puppeteer from 'puppeteer'; import fs from 'fs'; async function sleep(ms: number): Promise { return new Promise(resolve => setTimeout(resolve, ms)); } async function main() { const browser = await puppeteer.launch({ headless: true, args: ['--no-sandbox', '--disable-setuid-sandbox'], }); const page = await browser.newPage(); await page.setViewport({ width: 1920, height: 1080 }); // Capture ES API responses let allProductData: any[] = []; page.on('response', async (res) => { const url = res.url(); if (url.includes('gapcommerceapi.com/product/search') && res.status() === 200) { try { const json = await res.json(); const products = json.hits?.hits?.map((h: any) => h._source) || []; allProductData = allProductData.concat(products); console.log('Captured ' + products.length + ' products (total: ' + allProductData.length + ')'); } catch {} } }); console.log('Loading /shop page to capture product data...\n'); await page.goto('https://shop.bestdispensary.com/shop', { waitUntil: 'networkidle2', timeout: 60000 }); await sleep(3000); // Bypass age gate const ageGate = await page.$('[data-testid="age-gate-modal"]'); if (ageGate) { const btn = await page.$('[data-testid="age-gate-submit-button"]'); if (btn) await btn.click(); await sleep(2000); } // Click load more many times to get all products console.log('\nClicking Load More to capture all products...'); for (let i = 0; i < 50; i++) { const btn = await page.$('button.collection__load-more'); if (!btn) { console.log('No more Load More button'); break; } const isVisible = await page.evaluate((b) => { const rect = b.getBoundingClientRect(); return rect.width > 0 && rect.height > 0; }, btn); if (!isVisible) { console.log('Load More not visible'); break; } await btn.click(); await sleep(1500); console.log('Click ' + (i+1) + ': ' + allProductData.length + ' total products'); } console.log('\n=== RESULTS ===\n'); console.log('Total products captured: ' + allProductData.length); if (allProductData.length > 0) { // Dedupe by some ID const seen = new Set(); const unique = allProductData.filter(p => { const id = p.id || p.productId || p.name; if (seen.has(id)) return false; seen.add(id); return true; }); console.log('Unique products: ' + unique.length); console.log('\n=== PRODUCT FIELDS ===\n'); console.log(Object.keys(unique[0]).sort().join('\n')); console.log('\n=== SAMPLE PRODUCT ===\n'); console.log(JSON.stringify(unique[0], null, 2)); // Save to file fs.writeFileSync('/tmp/treez-products.json', JSON.stringify(unique, null, 2)); console.log('\nSaved to /tmp/treez-products.json'); } await browser.close(); } main();