import puppeteer from 'puppeteer'; async function sleep(ms: number): Promise { return new Promise(resolve => setTimeout(resolve, ms)); } async function main() { const browser = await puppeteer.launch({ headless: true, args: ['--no-sandbox', '--disable-setuid-sandbox'], }); const page = await browser.newPage(); await page.setViewport({ width: 1920, height: 1080 }); await page.setRequestInterception(true); page.on('request', (req) => { if (['image', 'font', 'media'].includes(req.resourceType())) { req.abort(); } else { req.continue(); } }); await page.goto('https://shop.bestdispensary.com/brands', { waitUntil: 'networkidle2', timeout: 60000 }); await sleep(3000); // Bypass age gate const ageGate = await page.$('[data-testid="age-gate-modal"]'); if (ageGate) { console.log('Bypassing age gate...'); const btn = await page.$('[data-testid="age-gate-submit-button"]'); if (btn) await btn.click(); await sleep(2000); } // Click "LOAD MORE" until all brands are loaded console.log('Loading all brands...\n'); let loadMoreClicks = 0; while (true) { const loadMoreBtn = await page.$('button.collection__load-more'); if (!loadMoreBtn) { console.log('No more "Load More" button - all brands loaded!'); break; } const isVisible = await page.evaluate((btn) => { const rect = btn.getBoundingClientRect(); return rect.width > 0 && rect.height > 0; }, loadMoreBtn); if (!isVisible) { console.log('Load More button not visible - all brands loaded!'); break; } await loadMoreBtn.click(); loadMoreClicks++; await sleep(1500); const brandCount = await page.evaluate(() => document.querySelectorAll('.brands-page__list a[href*="/brand/"]').length ); console.log(` Click ${loadMoreClicks}: ${brandCount} brands loaded`); if (loadMoreClicks > 20) break; // Safety limit } // Get all brands const brands = await page.evaluate(() => { const results: { name: string; href: string }[] = []; document.querySelectorAll('.brands-page__list a[href*="/brand/"]').forEach((a: Element) => { const href = a.getAttribute('href') || ''; const name = a.textContent?.trim() || ''; if (name && href) { results.push({ name, href }); } }); return results; }); console.log('\n' + '='.repeat(60)); console.log(`TOTAL BRANDS: ${brands.length}`); console.log('='.repeat(60)); // Visit each brand and count products console.log('\nCounting products per brand...\n'); const results: { brand: string; products: number }[] = []; for (let i = 0; i < brands.length; i++) { const brand = brands[i]; const brandUrl = `https://shop.bestdispensary.com${brand.href}`; try { await page.goto(brandUrl, { waitUntil: 'networkidle2', timeout: 30000 }); await sleep(1000); // Click load more on brand page too for (let j = 0; j < 10; j++) { const loadMore = await page.$('button.collection__load-more'); if (!loadMore) break; const isVisible = await page.evaluate((btn) => { const rect = btn.getBoundingClientRect(); return rect.width > 0 && rect.height > 0; }, loadMore); if (!isVisible) break; await loadMore.click(); await sleep(1000); } const productCount = await page.evaluate(() => { const seen = new Set(); document.querySelectorAll('a[href*="/product/"]').forEach((a: Element) => { const href = a.getAttribute('href'); if (href) seen.add(href); }); return seen.size; }); results.push({ brand: brand.name, products: productCount }); console.log(`${(i+1).toString().padStart(3)}. ${brand.name}: ${productCount} products`); } catch (err: any) { console.log(`${(i+1).toString().padStart(3)}. ${brand.name}: ERROR`); results.push({ brand: brand.name, products: 0 }); } } // Summary const totalProducts = results.reduce((sum, r) => sum + r.products, 0); const brandsWithProducts = results.filter(r => r.products > 0).length; console.log('\n' + '='.repeat(60)); console.log('SUMMARY'); console.log('='.repeat(60)); console.log(`Total brands: ${brands.length}`); console.log(`Brands with products: ${brandsWithProducts}`); console.log(`Total products: ${totalProducts}`); // Top brands by product count console.log('\nTop 20 brands by product count:'); results .sort((a, b) => b.products - a.products) .slice(0, 20) .forEach((r, i) => console.log(` ${i+1}. ${r.brand}: ${r.products}`)); await browser.close(); } main().catch(console.error);