chore: bump task worker version comment
Force new git SHA to avoid CI scientific notation bug. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
157
backend/scripts/test-brands-load-all.ts
Normal file
157
backend/scripts/test-brands-load-all.ts
Normal file
@@ -0,0 +1,157 @@
|
||||
import puppeteer from 'puppeteer';
|
||||
|
||||
async function sleep(ms: number): Promise<void> {
|
||||
return new Promise(resolve => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const browser = await puppeteer.launch({
|
||||
headless: true,
|
||||
args: ['--no-sandbox', '--disable-setuid-sandbox'],
|
||||
});
|
||||
|
||||
const page = await browser.newPage();
|
||||
await page.setViewport({ width: 1920, height: 1080 });
|
||||
|
||||
await page.setRequestInterception(true);
|
||||
page.on('request', (req) => {
|
||||
if (['image', 'font', 'media'].includes(req.resourceType())) {
|
||||
req.abort();
|
||||
} else {
|
||||
req.continue();
|
||||
}
|
||||
});
|
||||
|
||||
await page.goto('https://shop.bestdispensary.com/brands', {
|
||||
waitUntil: 'networkidle2',
|
||||
timeout: 60000
|
||||
});
|
||||
await sleep(3000);
|
||||
|
||||
// Bypass age gate
|
||||
const ageGate = await page.$('[data-testid="age-gate-modal"]');
|
||||
if (ageGate) {
|
||||
console.log('Bypassing age gate...');
|
||||
const btn = await page.$('[data-testid="age-gate-submit-button"]');
|
||||
if (btn) await btn.click();
|
||||
await sleep(2000);
|
||||
}
|
||||
|
||||
// Click "LOAD MORE" until all brands are loaded
|
||||
console.log('Loading all brands...\n');
|
||||
|
||||
let loadMoreClicks = 0;
|
||||
while (true) {
|
||||
const loadMoreBtn = await page.$('button.collection__load-more');
|
||||
if (!loadMoreBtn) {
|
||||
console.log('No more "Load More" button - all brands loaded!');
|
||||
break;
|
||||
}
|
||||
|
||||
const isVisible = await page.evaluate((btn) => {
|
||||
const rect = btn.getBoundingClientRect();
|
||||
return rect.width > 0 && rect.height > 0;
|
||||
}, loadMoreBtn);
|
||||
|
||||
if (!isVisible) {
|
||||
console.log('Load More button not visible - all brands loaded!');
|
||||
break;
|
||||
}
|
||||
|
||||
await loadMoreBtn.click();
|
||||
loadMoreClicks++;
|
||||
await sleep(1500);
|
||||
|
||||
const brandCount = await page.evaluate(() =>
|
||||
document.querySelectorAll('.brands-page__list a[href*="/brand/"]').length
|
||||
);
|
||||
console.log(` Click ${loadMoreClicks}: ${brandCount} brands loaded`);
|
||||
|
||||
if (loadMoreClicks > 20) break; // Safety limit
|
||||
}
|
||||
|
||||
// Get all brands
|
||||
const brands = await page.evaluate(() => {
|
||||
const results: { name: string; href: string }[] = [];
|
||||
document.querySelectorAll('.brands-page__list a[href*="/brand/"]').forEach((a: Element) => {
|
||||
const href = a.getAttribute('href') || '';
|
||||
const name = a.textContent?.trim() || '';
|
||||
if (name && href) {
|
||||
results.push({ name, href });
|
||||
}
|
||||
});
|
||||
return results;
|
||||
});
|
||||
|
||||
console.log('\n' + '='.repeat(60));
|
||||
console.log(`TOTAL BRANDS: ${brands.length}`);
|
||||
console.log('='.repeat(60));
|
||||
|
||||
// Visit each brand and count products
|
||||
console.log('\nCounting products per brand...\n');
|
||||
|
||||
const results: { brand: string; products: number }[] = [];
|
||||
|
||||
for (let i = 0; i < brands.length; i++) {
|
||||
const brand = brands[i];
|
||||
const brandUrl = `https://shop.bestdispensary.com${brand.href}`;
|
||||
|
||||
try {
|
||||
await page.goto(brandUrl, { waitUntil: 'networkidle2', timeout: 30000 });
|
||||
await sleep(1000);
|
||||
|
||||
// Click load more on brand page too
|
||||
for (let j = 0; j < 10; j++) {
|
||||
const loadMore = await page.$('button.collection__load-more');
|
||||
if (!loadMore) break;
|
||||
|
||||
const isVisible = await page.evaluate((btn) => {
|
||||
const rect = btn.getBoundingClientRect();
|
||||
return rect.width > 0 && rect.height > 0;
|
||||
}, loadMore);
|
||||
|
||||
if (!isVisible) break;
|
||||
await loadMore.click();
|
||||
await sleep(1000);
|
||||
}
|
||||
|
||||
const productCount = await page.evaluate(() => {
|
||||
const seen = new Set<string>();
|
||||
document.querySelectorAll('a[href*="/product/"]').forEach((a: Element) => {
|
||||
const href = a.getAttribute('href');
|
||||
if (href) seen.add(href);
|
||||
});
|
||||
return seen.size;
|
||||
});
|
||||
|
||||
results.push({ brand: brand.name, products: productCount });
|
||||
console.log(`${(i+1).toString().padStart(3)}. ${brand.name}: ${productCount} products`);
|
||||
|
||||
} catch (err: any) {
|
||||
console.log(`${(i+1).toString().padStart(3)}. ${brand.name}: ERROR`);
|
||||
results.push({ brand: brand.name, products: 0 });
|
||||
}
|
||||
}
|
||||
|
||||
// Summary
|
||||
const totalProducts = results.reduce((sum, r) => sum + r.products, 0);
|
||||
const brandsWithProducts = results.filter(r => r.products > 0).length;
|
||||
|
||||
console.log('\n' + '='.repeat(60));
|
||||
console.log('SUMMARY');
|
||||
console.log('='.repeat(60));
|
||||
console.log(`Total brands: ${brands.length}`);
|
||||
console.log(`Brands with products: ${brandsWithProducts}`);
|
||||
console.log(`Total products: ${totalProducts}`);
|
||||
|
||||
// Top brands by product count
|
||||
console.log('\nTop 20 brands by product count:');
|
||||
results
|
||||
.sort((a, b) => b.products - a.products)
|
||||
.slice(0, 20)
|
||||
.forEach((r, i) => console.log(` ${i+1}. ${r.brand}: ${r.products}`));
|
||||
|
||||
await browser.close();
|
||||
}
|
||||
|
||||
main().catch(console.error);
|
||||
Reference in New Issue
Block a user