58 lines
2.0 KiB
TypeScript
58 lines
2.0 KiB
TypeScript
import { scrapeCategoryPlaywright } from './src/services/scraper-playwright';
|
|
import { pool } from './src/db/migrate';
|
|
import { getRandomProxy, getStateProxy, getProxyLocationStats } from './src/utils/proxyManager';
|
|
|
|
async function testStealthScraper() {
|
|
console.log('🧪 Testing stealth scraper on Sol Flower Sun City\n');
|
|
|
|
const categoryUrl = 'https://dutchie.com/dispensary/sol-flower-dispensary/shop/flower';
|
|
const categoryName = 'Flower';
|
|
const state = 'Arizona';
|
|
|
|
console.log(`📂 Category: ${categoryName}`);
|
|
console.log(`🔗 URL: ${categoryUrl}`);
|
|
console.log(`📍 State: ${state}\n`);
|
|
|
|
try {
|
|
// Show proxy location distribution
|
|
console.log('📊 Proxy location distribution:');
|
|
const stats = await getProxyLocationStats();
|
|
console.table(stats.slice(0, 10));
|
|
console.log('');
|
|
|
|
// Get an active residential proxy from database
|
|
console.log('🔍 Fetching active residential proxy...');
|
|
const proxy = await getRandomProxy();
|
|
|
|
if (proxy) {
|
|
console.log(`✅ Using proxy: ${proxy.server}`);
|
|
console.log(` Credentials: ${proxy.username ? 'Yes' : 'No'}\n`);
|
|
} else {
|
|
console.log('⚠️ No active proxies found, proceeding without proxy\n');
|
|
}
|
|
|
|
const products = await scrapeCategoryPlaywright(categoryUrl, categoryName, state, proxy);
|
|
|
|
console.log(`\n✅ Scrape completed!`);
|
|
console.log(`📦 Found ${products.length} products\n`);
|
|
|
|
if (products.length > 0) {
|
|
console.log('🎉 SUCCESS! Cloudflare bypass worked!\n');
|
|
console.log('First 5 products:');
|
|
products.slice(0, 5).forEach((product, i) => {
|
|
console.log(` ${i + 1}. ${product.name}`);
|
|
if (product.brand) console.log(` Brand: ${product.brand}`);
|
|
if (product.price) console.log(` Price: $${product.price}`);
|
|
});
|
|
} else {
|
|
console.log('⚠️ No products found - may need more stealth adjustments');
|
|
}
|
|
} catch (error) {
|
|
console.error('❌ Error:', error);
|
|
} finally {
|
|
await pool.end();
|
|
}
|
|
}
|
|
|
|
testStealthScraper();
|